Add --max-rps throttle for exchange-graph and exchange-ews #76

Merged
jcoffey-dev merged 1 commits from feat/migrate-max-rps into main 2026-11-13 01:20:37 +00:00
10 changed files with 256 additions and 5 deletions

No files matched your search

+5
View File
@@ -17,6 +17,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
`INBUXA_MIGRATE_API_KEY_FILE`, and so on). One trailing newline is ignored;
setting both a variable and its `_FILE` form is an error. This keeps a
secret out of both the environment and the command line.
- `import exchange-graph` and `import exchange-ews` take `--max-rps` to cap
how many requests start per second across all worker connections -- a
proactive throttle to stay under a rate-limited Microsoft 365 tenant's
limit, alongside the existing honoring of 429/`Retry-After`. Off by
default (uncapped).
## [27.1-rc.12]
+13 -2
View File
@@ -266,7 +266,8 @@ inbuxa-migrate import exchange-ews \
(--auth-basic <USER> [--auth-password <PASS>] \
| --auth-bearer [TOKEN] [--ews-tenant <T> --ews-client-id <ID> \
(--ews-device-code | --ews-client-secret <SECRET>)]) \
[--ews-connections <1..8>] [--ews-getitem-batch <N>] [--ews-getitem-batch-mib <MIB>] \
[--ews-connections <1..8>] [--max-rps <RPS>] \
[--ews-getitem-batch <N>] [--ews-getitem-batch-mib <MIB>] \
[--ews-attachment-batch <N>] \
[--ews-no-syncfolderitems] \
<ARCHIVE>
@@ -283,6 +284,11 @@ time. The byte cap applies where Exchange reports each item's size, which it
does on a full listing; items found through an incremental sync are batched
by count.
The server throttles on its own, and a 429 or 503 is honored (its
`Retry-After`, then a shared backoff across the workers). To stay under a
known limit in the first place, `--max-rps` caps how many requests start per
second across all connections; leave it off to run uncapped.
For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from October 1, 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on April 1, 2027 it is switched off for every tenant. On-premises Exchange Server is not affected.
### Microsoft Exchange (Graph)
@@ -294,13 +300,18 @@ inbuxa-migrate import exchange-graph \
[--mailbox-kind primary|archive] \
[--objects mail,calendar,contacts] \
[--event-body-format text|html] \
[--graph-connections <1..16>] [--top <1..1000>] \
[--graph-connections <1..16>] [--max-rps <RPS>] [--top <1..1000>] \
<ARCHIVE>
```
Imports a mailbox from Exchange Online through Microsoft Graph. Without
`--access-token` it signs in with the interactive device-code flow. `public-folders` is rejected here: Graph does not expose public folders, so they can only be imported with `exchange-ews`, which for Exchange Online is subject to the retirement described above.
Graph throttles per app and per mailbox, and a 429 or 503 is honored (its
`Retry-After`, then a shared backoff across the workers). `--max-rps` caps how
many requests start per second across all connections, to stay under a tenant's
limit before it pushes back; leave it off to run uncapped.
With `--app-only` the app signs in as itself, with its client secret, and
reads the mailbox `--user` names. No one signs in, so a whole tenant moves
with one app, in a [plan](#migrate-many-accounts) with `--user {email}`.
+79
View File
@@ -1820,6 +1820,14 @@ pub struct ExchangeEwsImportArgs {
)]
ews_connections: usize,
#[arg(
long,
value_name = "RPS",
value_parser = parse_max_rps,
help = "Cap requests per second across all connections (proactive throttle for a rate-limited server)"
)]
max_rps: Option<f64>,
#[arg(
long,
value_name = "N",
@@ -1884,6 +1892,7 @@ fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result<Action, Er
mailbox_kind,
auth,
ews_connections,
max_rps: args.max_rps,
getitem_batch: args.ews_getitem_batch.max(1),
getitem_batch_bytes: args
.ews_getitem_batch_mib
@@ -2036,6 +2045,14 @@ pub struct ExchangeGraphImportArgs {
)]
graph_connections: usize,
#[arg(
long,
value_name = "RPS",
value_parser = parse_max_rps,
help = "Cap requests per second across all connections (proactive throttle for a rate-limited tenant)"
)]
max_rps: Option<f64>,
#[arg(
long,
value_name = "N",
@@ -2103,6 +2120,14 @@ fn parse_event_body_format(s: &str) -> Result<EventBodyFormat, String> {
EventBodyFormat::parse(s).map_err(|e| e.to_string())
}
/// A requests-per-second cap: a positive, finite number.
fn parse_max_rps(s: &str) -> Result<f64, String> {
match s.parse::<f64>() {
Ok(v) if v.is_finite() && v > 0.0 => Ok(v),
_ => Err("expected a positive number of requests per second".to_owned()),
}
}
fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action, Error> {
let common = common_config(&args.global, args.archive.clone());
let mailbox_kind = args.mailbox_kind;
@@ -2134,6 +2159,7 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
surfaces,
event_body_format,
graph_connections,
max_rps: args.max_rps,
top,
exception_window_years: args.exception_window_years,
contact_photos: !args.skip_contact_photos,
@@ -2282,6 +2308,59 @@ mod tests {
assert!(matches!(cli.resolve().unwrap(), Action::Man));
}
#[test]
fn graph_max_rps_is_off_by_default_and_parsed_when_given() {
assert_eq!(graph_config(&[]).unwrap().max_rps, None);
assert_eq!(
graph_config(&["--max-rps", "8.5"]).unwrap().max_rps,
Some(8.5)
);
}
#[test]
fn graph_max_rps_must_be_positive() {
for bad in ["0", "-3", "abc", "inf"] {
assert!(
Cli::try_parse_from([
"inbuxa-migrate",
"import",
"exchange-graph",
"--access-token",
"tok",
"--max-rps",
bad,
"archive.sqlite",
])
.is_err(),
"--max-rps {bad} should be rejected"
);
}
}
#[test]
fn ews_max_rps_is_off_by_default_and_parsed_when_given() {
let config = |extra: &[&str]| -> EwsImportConfig {
let mut argv = vec![
"inbuxa-migrate",
"import",
"exchange-ews",
"--url",
"https://ews.example.com/EWS/Exchange.asmx",
"--auth-bearer",
"tok",
];
argv.extend_from_slice(extra);
argv.push("archive.sqlite");
let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected: {e}"));
match cli.resolve().unwrap() {
Action::ImportExchangeEws(_, c) => c,
_ => panic!("expected ImportExchangeEws"),
}
};
assert_eq!(config(&[]).max_rps, None);
assert_eq!(config(&["--max-rps", "4"]).max_rps, Some(4.0));
}
#[test]
fn parse_duration_reads_units_and_rejects_nonsense() {
use std::time::Duration;
+10 -1
View File
@@ -19,7 +19,7 @@ use crate::exchange_ews::retry::{FaultDisposition, classify_fault, classify_http
use crate::exchange_ews::soap::{EnvelopeOptions, soap_action, wrap_envelope};
use crate::exchange_ews::types::ServerVersion;
use crate::jmap::http::{Auth, RetryPolicy, retry_after_header};
use crate::jmap::retry::{self, Disposition, RateLimitState};
use crate::jmap::retry::{self, Disposition, Pacer, RateLimitState};
use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::net::{CertOverride, tls, with_timeouts};
@@ -36,6 +36,7 @@ struct Inner {
affinity_cookie: Mutex<Option<String>>,
retry: RetryPolicy,
rate_limit: RateLimitState,
pace: Pacer,
version: Mutex<ServerVersion>,
log_level: AtomicU8,
retries_total: AtomicU64,
@@ -90,6 +91,7 @@ impl EwsClient {
affinity_cookie: Mutex::new(None),
retry,
rate_limit: RateLimitState::new(),
pace: Pacer::new(),
version: Mutex::new(ServerVersion::Exchange2013Sp1),
log_level: AtomicU8::new(LEVEL_DEFAULT),
retries_total: AtomicU64::new(0),
@@ -124,6 +126,12 @@ impl EwsClient {
.store(logger.level(), Ordering::Relaxed);
}
/// Cap how often a request may start, in requests per second, across all
/// worker connections. `None` or a non-positive rate leaves it uncapped.
pub fn set_max_rps(&self, max_rps: Option<f64>) {
self.inner.pace.set_per_second(max_rps);
}
pub fn logger(&self) -> Logger {
Logger::new(self.inner.log_level.load(Ordering::Relaxed))
}
@@ -209,6 +217,7 @@ impl EwsClient {
let mut attempt: u32 = 0;
loop {
self.inner.rate_limit.cooldown().wait();
self.inner.pace.wait();
let sent_version = self.server_version();
let envelope = self.wrap(body);
let outcome = self.one_attempt(url, &envelope, action);
+10 -1
View File
@@ -18,7 +18,7 @@ use ureq::{ResponseExt, http::Uri};
use crate::exchange_graph::error::GraphError;
use crate::exchange_graph::retry::{HttpClass, classify_http_status, is_throttled};
use crate::jmap::http::{RetryPolicy, cross_host, retry_after_header};
use crate::jmap::retry::{self, RateLimitState};
use crate::jmap::retry::{self, Pacer, RateLimitState};
use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::net::{CertOverride, tls, with_timeouts};
@@ -68,6 +68,7 @@ struct Inner {
bearer: Mutex<String>,
retry: RetryPolicy,
rate_limit: RateLimitState,
pace: Pacer,
log_level: AtomicU8,
retries_total: AtomicU64,
retry_after_sleeps: AtomicU64,
@@ -122,6 +123,7 @@ impl GraphClient {
bearer: Mutex::new(bearer),
retry,
rate_limit: RateLimitState::new(),
pace: Pacer::new(),
log_level: AtomicU8::new(LEVEL_DEFAULT),
retries_total: AtomicU64::new(0),
retry_after_sleeps: AtomicU64::new(0),
@@ -145,6 +147,12 @@ impl GraphClient {
.store(logger.level(), Ordering::Relaxed);
}
/// Cap how often a request may start, in requests per second, across all
/// worker connections. `None` or a non-positive rate leaves it uncapped.
pub fn set_max_rps(&self, max_rps: Option<f64>) {
self.inner.pace.set_per_second(max_rps);
}
fn logger(&self) -> Logger {
Logger::new(self.inner.log_level.load(Ordering::Relaxed))
}
@@ -210,6 +218,7 @@ impl GraphClient {
let mut attempt: u32 = 0;
loop {
self.inner.rate_limit.cooldown().wait();
self.inner.pace.wait();
self.inner.requests_total.fetch_add(1, Ordering::Relaxed);
let outcome = self.one_attempt(method, url, accept, extra_prefer);
match outcome {
+121 -1
View File
@@ -6,7 +6,7 @@
use std::cell::Cell;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use crate::jmap::http::RetryPolicy;
@@ -178,6 +178,69 @@ impl RateLimitState {
}
}
/// A proactive request-rate cap, shared across worker threads. It spaces the
/// *start* of successive requests at least one interval apart, so a known
/// provider limit (Microsoft Graph, Google) is kept to in the first place,
/// rather than reactively, after the server returns 429 and the [`Cooldown`]
/// engages. The interval is set once, after the client is built (like a
/// logger), and a zero interval disables it so there is no cost on the hot
/// path when no cap is asked for. Requests in flight are not limited — only
/// how often a new one may begin — so it composes with the connection count.
#[derive(Debug, Default)]
pub struct Pacer {
/// Minimum spacing between request starts, in nanoseconds; 0 disables.
min_interval_nanos: AtomicU64,
next: Mutex<Option<Instant>>,
}
impl Pacer {
pub fn new() -> Pacer {
Pacer {
min_interval_nanos: AtomicU64::new(0),
next: Mutex::new(None),
}
}
/// Set the cap from a requests-per-second figure. `None`, zero, or a
/// non-finite rate disables pacing. A later call replaces the cap.
pub fn set_per_second(&self, max_rps: Option<f64>) {
let nanos = match max_rps {
Some(rps) if rps.is_finite() && rps > 0.0 => {
(1.0e9 / rps).round().clamp(1.0, u64::MAX as f64) as u64
}
_ => 0,
};
self.min_interval_nanos.store(nanos, Ordering::Relaxed);
}
pub fn is_enabled(&self) -> bool {
self.min_interval_nanos.load(Ordering::Relaxed) != 0
}
/// Block until this thread may start its request, then reserve its slot.
/// Holding the lock across the sleep is deliberate: it serializes the
/// reservations, so N waiters leave one interval apart instead of all at
/// once. An idle gap does not bank credit — the next request starts now,
/// never a burst.
pub fn wait(&self) {
let nanos = self.min_interval_nanos.load(Ordering::Relaxed);
if nanos == 0 {
return;
}
let interval = Duration::from_nanos(nanos);
let mut next = self.next.lock().unwrap_or_else(|p| p.into_inner());
let now = Instant::now();
let start = match *next {
Some(t) if t > now => {
std::thread::sleep(t - now);
t
}
_ => now,
};
*next = Some(start + interval);
}
}
pub fn throttle_backoff(policy: &RetryPolicy, level: u32) -> Duration {
let shift = level.saturating_sub(1).min(20);
let scaled = THROTTLE_BASE
@@ -340,4 +403,61 @@ mod tests {
assert_eq!(d, Duration::from_millis(750));
assert_eq!(s.level(), 1);
}
#[test]
fn pacer_disabled_by_default_and_by_zero_or_none() {
let p = Pacer::new();
assert!(!p.is_enabled());
p.set_per_second(Some(10.0));
assert!(p.is_enabled());
p.set_per_second(None);
assert!(!p.is_enabled());
p.set_per_second(Some(0.0));
assert!(!p.is_enabled());
p.set_per_second(Some(-5.0));
assert!(!p.is_enabled());
p.set_per_second(Some(f64::NAN));
assert!(!p.is_enabled());
}
#[test]
fn pacer_spaces_successive_waits_by_the_interval() {
// 50 rps -> 20ms apart. The first wait returns at once (no prior
// slot); the next few are each spaced ~one interval on.
let p = Pacer::new();
p.set_per_second(Some(50.0));
let start = Instant::now();
for _ in 0..4 {
p.wait();
}
// Three gaps of 20ms = 60ms minimum after the immediate first.
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(55),
"four 50rps waits took {elapsed:?}, expected >= ~60ms"
);
}
#[test]
fn pacer_wait_is_immediate_when_disabled() {
let p = Pacer::new();
let start = Instant::now();
for _ in 0..1000 {
p.wait();
}
assert!(start.elapsed() < Duration::from_millis(50));
}
#[test]
fn pacer_does_not_bank_credit_over_an_idle_gap() {
// After a pause longer than the interval, the next request still
// starts immediately (one slot), not a burst of pre-earned slots.
let p = Pacer::new();
p.set_per_second(Some(100.0)); // 10ms
p.wait(); // takes a slot now
std::thread::sleep(Duration::from_millis(40));
let start = Instant::now();
p.wait(); // next is in the past -> immediate
assert!(start.elapsed() < Duration::from_millis(5));
}
}
@@ -38,6 +38,7 @@ pub struct EwsImportConfig {
pub mailbox_kind: MailboxKind,
pub auth: EwsAuth,
pub ews_connections: usize,
pub max_rps: Option<f64>,
pub getitem_batch: usize,
/// Byte cap for one GetItem batch; see `sync::batch`.
pub getitem_batch_bytes: u64,
@@ -85,6 +86,7 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
certs.narrowed_to(&discovery.ews_url),
);
client.set_logger(logger);
client.set_max_rps(config.max_rps);
if matches!(config.mailbox_kind, MailboxKind::PublicFolders) {
client.set_anchor_mailbox(None);
} else {
@@ -50,6 +50,7 @@ pub struct GraphImportConfig {
pub surfaces: Surfaces,
pub event_body_format: EventBodyFormat,
pub graph_connections: usize,
pub max_rps: Option<f64>,
pub top: usize,
pub exception_window_years: i32,
pub contact_photos: bool,
@@ -88,6 +89,7 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
CertOverride::for_url(common.allow_invalid_certs, &config.api_base),
);
client.set_logger(logger);
client.set_max_rps(config.max_rps);
let endpoints = resolve_endpoints(&config, &client)?;
let principal = resolve_principal(&client, &endpoints, acquired.upn.as_deref())?;
@@ -559,6 +561,7 @@ mod tests {
surfaces: Surfaces::ALL,
event_body_format: EventBodyFormat::Text,
graph_connections: 4,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: true,
+12
View File
@@ -594,6 +594,7 @@ fn integration_dry_run_against_mock_server_lists_three_surfaces() {
surfaces: Surfaces::ALL,
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -707,6 +708,7 @@ fn integration_full_run_mail_only_imports_mime_via_value() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -807,6 +809,7 @@ fn integration_duplicate_message_id_does_not_abort_run() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -899,6 +902,7 @@ fn integration_full_run_is_convergent_on_second_invocation() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -986,6 +990,7 @@ fn source_change_protection_refuses_a_different_account() {
surfaces: Surfaces::ALL,
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -1056,6 +1061,7 @@ fn make_config(
surfaces,
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -1776,6 +1782,7 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -1850,6 +1857,7 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -1919,6 +1927,7 @@ fn allow_source_change_permits_overwriting_a_different_account() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -2004,6 +2013,7 @@ fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() {
surfaces: Surfaces::ALL,
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -2176,6 +2186,7 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
@@ -3139,6 +3150,7 @@ fn app_only_signs_in_as_the_app_and_reads_the_named_mailbox() {
surfaces: surfaces("mail"),
event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: false,
+1
View File
@@ -292,6 +292,7 @@ fn graph_import_survives_a_full_export_to_stalwart() {
surfaces: Surfaces::ALL,
event_body_format: EventBodyFormat::Text,
graph_connections: 2,
max_rps: None,
top: 100,
exception_window_years: 5,
contact_photos: true,