From 763fdca425d0ae85ac82100d67c273efdb1116d6 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 12 Nov 2026 16:37:24 -0700 Subject: [PATCH] Add --max-rps throttle for exchange-graph and exchange-ews Microsoft 365 throttles per app and per mailbox. The clients already honor a 429/503 reactively (its Retry-After, then a shared backoff across the workers), but nothing let an operator stay under a known limit in the first place. --max-rps caps how many requests may start per second across all worker connections, a proactive throttle. It is off by default (uncapped) and composes with the connection count: requests in flight are not limited, only how often a new one begins. A reusable Pacer lives next to Cooldown in jmap::retry: the interval is set once after the client is built (like the logger), a zero interval is free on the hot path, and an idle gap banks no burst credit. Both M365 clients call it at the top of their request loop. Surfaced on the two HTTP-request importers, where requests-per-second is the natural knob; for Gmail the lever stays --imap-connections. --- CHANGELOG.md | 5 + docs/usage.md | 15 ++- src/cli.rs | 79 ++++++++++++ src/exchange_ews/client.rs | 11 +- src/exchange_graph/client.rs | 11 +- src/jmap/retry.rs | 122 +++++++++++++++++- src/sync/import_exchange_ews/coordinator.rs | 2 + src/sync/import_exchange_graph/coordinator.rs | 3 + tests/mock_exchange_graph.rs | 12 ++ tests/sync_exchange_graph.rs | 1 + 10 files changed, 256 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eaf4ce..a584bcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a `INBUXA_MIGRATE_API_KEY_FILE`, and so on). One trailing newline is ignored; setting both a variable and its `_FILE` form is an error. This keeps a secret out of both the environment and the command line. +- `import exchange-graph` and `import exchange-ews` take `--max-rps` to cap + how many requests start per second across all worker connections -- a + proactive throttle to stay under a rate-limited Microsoft 365 tenant's + limit, alongside the existing honoring of 429/`Retry-After`. Off by + default (uncapped). ## [27.1-rc.12] diff --git a/docs/usage.md b/docs/usage.md index d247dcf..6f7421d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -266,7 +266,8 @@ inbuxa-migrate import exchange-ews \ (--auth-basic [--auth-password ] \ | --auth-bearer [TOKEN] [--ews-tenant --ews-client-id \ (--ews-device-code | --ews-client-secret )]) \ - [--ews-connections <1..8>] [--ews-getitem-batch ] [--ews-getitem-batch-mib ] \ + [--ews-connections <1..8>] [--max-rps ] \ + [--ews-getitem-batch ] [--ews-getitem-batch-mib ] \ [--ews-attachment-batch ] \ [--ews-no-syncfolderitems] \ @@ -283,6 +284,11 @@ time. The byte cap applies where Exchange reports each item's size, which it does on a full listing; items found through an incremental sync are batched by count. +The server throttles on its own, and a 429 or 503 is honored (its +`Retry-After`, then a shared backoff across the workers). To stay under a +known limit in the first place, `--max-rps` caps how many requests start per +second across all connections; leave it off to run uncapped. + For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from October 1, 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on April 1, 2027 it is switched off for every tenant. On-premises Exchange Server is not affected. ### Microsoft Exchange (Graph) @@ -294,13 +300,18 @@ inbuxa-migrate import exchange-graph \ [--mailbox-kind primary|archive] \ [--objects mail,calendar,contacts] \ [--event-body-format text|html] \ - [--graph-connections <1..16>] [--top <1..1000>] \ + [--graph-connections <1..16>] [--max-rps ] [--top <1..1000>] \ ``` Imports a mailbox from Exchange Online through Microsoft Graph. Without `--access-token` it signs in with the interactive device-code flow. `public-folders` is rejected here: Graph does not expose public folders, so they can only be imported with `exchange-ews`, which for Exchange Online is subject to the retirement described above. +Graph throttles per app and per mailbox, and a 429 or 503 is honored (its +`Retry-After`, then a shared backoff across the workers). `--max-rps` caps how +many requests start per second across all connections, to stay under a tenant's +limit before it pushes back; leave it off to run uncapped. + With `--app-only` the app signs in as itself, with its client secret, and reads the mailbox `--user` names. No one signs in, so a whole tenant moves with one app, in a [plan](#migrate-many-accounts) with `--user {email}`. diff --git a/src/cli.rs b/src/cli.rs index ed8c4f5..81f289e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1820,6 +1820,14 @@ pub struct ExchangeEwsImportArgs { )] ews_connections: usize, + #[arg( + long, + value_name = "RPS", + value_parser = parse_max_rps, + help = "Cap requests per second across all connections (proactive throttle for a rate-limited server)" + )] + max_rps: Option, + #[arg( long, value_name = "N", @@ -1884,6 +1892,7 @@ fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result, + #[arg( long, value_name = "N", @@ -2103,6 +2120,14 @@ fn parse_event_body_format(s: &str) -> Result { EventBodyFormat::parse(s).map_err(|e| e.to_string()) } +/// A requests-per-second cap: a positive, finite number. +fn parse_max_rps(s: &str) -> Result { + match s.parse::() { + Ok(v) if v.is_finite() && v > 0.0 => Ok(v), + _ => Err("expected a positive number of requests per second".to_owned()), + } +} + fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result { let common = common_config(&args.global, args.archive.clone()); let mailbox_kind = args.mailbox_kind; @@ -2134,6 +2159,7 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result EwsImportConfig { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "exchange-ews", + "--url", + "https://ews.example.com/EWS/Exchange.asmx", + "--auth-bearer", + "tok", + ]; + argv.extend_from_slice(extra); + argv.push("archive.sqlite"); + let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected: {e}")); + match cli.resolve().unwrap() { + Action::ImportExchangeEws(_, c) => c, + _ => panic!("expected ImportExchangeEws"), + } + }; + assert_eq!(config(&[]).max_rps, None); + assert_eq!(config(&["--max-rps", "4"]).max_rps, Some(4.0)); + } + #[test] fn parse_duration_reads_units_and_rejects_nonsense() { use std::time::Duration; diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs index 0d64d44..01f83e9 100644 --- a/src/exchange_ews/client.rs +++ b/src/exchange_ews/client.rs @@ -19,7 +19,7 @@ use crate::exchange_ews::retry::{FaultDisposition, classify_fault, classify_http use crate::exchange_ews::soap::{EnvelopeOptions, soap_action, wrap_envelope}; use crate::exchange_ews::types::ServerVersion; use crate::jmap::http::{Auth, RetryPolicy, retry_after_header}; -use crate::jmap::retry::{self, Disposition, RateLimitState}; +use crate::jmap::retry::{self, Disposition, Pacer, RateLimitState}; use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::net::{CertOverride, tls, with_timeouts}; @@ -36,6 +36,7 @@ struct Inner { affinity_cookie: Mutex>, retry: RetryPolicy, rate_limit: RateLimitState, + pace: Pacer, version: Mutex, log_level: AtomicU8, retries_total: AtomicU64, @@ -90,6 +91,7 @@ impl EwsClient { affinity_cookie: Mutex::new(None), retry, rate_limit: RateLimitState::new(), + pace: Pacer::new(), version: Mutex::new(ServerVersion::Exchange2013Sp1), log_level: AtomicU8::new(LEVEL_DEFAULT), retries_total: AtomicU64::new(0), @@ -124,6 +126,12 @@ impl EwsClient { .store(logger.level(), Ordering::Relaxed); } + /// Cap how often a request may start, in requests per second, across all + /// worker connections. `None` or a non-positive rate leaves it uncapped. + pub fn set_max_rps(&self, max_rps: Option) { + self.inner.pace.set_per_second(max_rps); + } + pub fn logger(&self) -> Logger { Logger::new(self.inner.log_level.load(Ordering::Relaxed)) } @@ -209,6 +217,7 @@ impl EwsClient { let mut attempt: u32 = 0; loop { self.inner.rate_limit.cooldown().wait(); + self.inner.pace.wait(); let sent_version = self.server_version(); let envelope = self.wrap(body); let outcome = self.one_attempt(url, &envelope, action); diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs index 14c13de..1123f82 100644 --- a/src/exchange_graph/client.rs +++ b/src/exchange_graph/client.rs @@ -18,7 +18,7 @@ use ureq::{ResponseExt, http::Uri}; use crate::exchange_graph::error::GraphError; use crate::exchange_graph::retry::{HttpClass, classify_http_status, is_throttled}; use crate::jmap::http::{RetryPolicy, cross_host, retry_after_header}; -use crate::jmap::retry::{self, RateLimitState}; +use crate::jmap::retry::{self, Pacer, RateLimitState}; use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::net::{CertOverride, tls, with_timeouts}; @@ -68,6 +68,7 @@ struct Inner { bearer: Mutex, retry: RetryPolicy, rate_limit: RateLimitState, + pace: Pacer, log_level: AtomicU8, retries_total: AtomicU64, retry_after_sleeps: AtomicU64, @@ -122,6 +123,7 @@ impl GraphClient { bearer: Mutex::new(bearer), retry, rate_limit: RateLimitState::new(), + pace: Pacer::new(), log_level: AtomicU8::new(LEVEL_DEFAULT), retries_total: AtomicU64::new(0), retry_after_sleeps: AtomicU64::new(0), @@ -145,6 +147,12 @@ impl GraphClient { .store(logger.level(), Ordering::Relaxed); } + /// Cap how often a request may start, in requests per second, across all + /// worker connections. `None` or a non-positive rate leaves it uncapped. + pub fn set_max_rps(&self, max_rps: Option) { + self.inner.pace.set_per_second(max_rps); + } + fn logger(&self) -> Logger { Logger::new(self.inner.log_level.load(Ordering::Relaxed)) } @@ -210,6 +218,7 @@ impl GraphClient { let mut attempt: u32 = 0; loop { self.inner.rate_limit.cooldown().wait(); + self.inner.pace.wait(); self.inner.requests_total.fetch_add(1, Ordering::Relaxed); let outcome = self.one_attempt(method, url, accept, extra_prefer); match outcome { diff --git a/src/jmap/retry.rs b/src/jmap/retry.rs index 3ea86cd..f988e44 100644 --- a/src/jmap/retry.rs +++ b/src/jmap/retry.rs @@ -6,7 +6,7 @@ use std::cell::Cell; use std::sync::Mutex; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::jmap::http::RetryPolicy; @@ -178,6 +178,69 @@ impl RateLimitState { } } +/// A proactive request-rate cap, shared across worker threads. It spaces the +/// *start* of successive requests at least one interval apart, so a known +/// provider limit (Microsoft Graph, Google) is kept to in the first place, +/// rather than reactively, after the server returns 429 and the [`Cooldown`] +/// engages. The interval is set once, after the client is built (like a +/// logger), and a zero interval disables it so there is no cost on the hot +/// path when no cap is asked for. Requests in flight are not limited — only +/// how often a new one may begin — so it composes with the connection count. +#[derive(Debug, Default)] +pub struct Pacer { + /// Minimum spacing between request starts, in nanoseconds; 0 disables. + min_interval_nanos: AtomicU64, + next: Mutex>, +} + +impl Pacer { + pub fn new() -> Pacer { + Pacer { + min_interval_nanos: AtomicU64::new(0), + next: Mutex::new(None), + } + } + + /// Set the cap from a requests-per-second figure. `None`, zero, or a + /// non-finite rate disables pacing. A later call replaces the cap. + pub fn set_per_second(&self, max_rps: Option) { + let nanos = match max_rps { + Some(rps) if rps.is_finite() && rps > 0.0 => { + (1.0e9 / rps).round().clamp(1.0, u64::MAX as f64) as u64 + } + _ => 0, + }; + self.min_interval_nanos.store(nanos, Ordering::Relaxed); + } + + pub fn is_enabled(&self) -> bool { + self.min_interval_nanos.load(Ordering::Relaxed) != 0 + } + + /// Block until this thread may start its request, then reserve its slot. + /// Holding the lock across the sleep is deliberate: it serializes the + /// reservations, so N waiters leave one interval apart instead of all at + /// once. An idle gap does not bank credit — the next request starts now, + /// never a burst. + pub fn wait(&self) { + let nanos = self.min_interval_nanos.load(Ordering::Relaxed); + if nanos == 0 { + return; + } + let interval = Duration::from_nanos(nanos); + let mut next = self.next.lock().unwrap_or_else(|p| p.into_inner()); + let now = Instant::now(); + let start = match *next { + Some(t) if t > now => { + std::thread::sleep(t - now); + t + } + _ => now, + }; + *next = Some(start + interval); + } +} + pub fn throttle_backoff(policy: &RetryPolicy, level: u32) -> Duration { let shift = level.saturating_sub(1).min(20); let scaled = THROTTLE_BASE @@ -340,4 +403,61 @@ mod tests { assert_eq!(d, Duration::from_millis(750)); assert_eq!(s.level(), 1); } + + #[test] + fn pacer_disabled_by_default_and_by_zero_or_none() { + let p = Pacer::new(); + assert!(!p.is_enabled()); + p.set_per_second(Some(10.0)); + assert!(p.is_enabled()); + p.set_per_second(None); + assert!(!p.is_enabled()); + p.set_per_second(Some(0.0)); + assert!(!p.is_enabled()); + p.set_per_second(Some(-5.0)); + assert!(!p.is_enabled()); + p.set_per_second(Some(f64::NAN)); + assert!(!p.is_enabled()); + } + + #[test] + fn pacer_spaces_successive_waits_by_the_interval() { + // 50 rps -> 20ms apart. The first wait returns at once (no prior + // slot); the next few are each spaced ~one interval on. + let p = Pacer::new(); + p.set_per_second(Some(50.0)); + let start = Instant::now(); + for _ in 0..4 { + p.wait(); + } + // Three gaps of 20ms = 60ms minimum after the immediate first. + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(55), + "four 50rps waits took {elapsed:?}, expected >= ~60ms" + ); + } + + #[test] + fn pacer_wait_is_immediate_when_disabled() { + let p = Pacer::new(); + let start = Instant::now(); + for _ in 0..1000 { + p.wait(); + } + assert!(start.elapsed() < Duration::from_millis(50)); + } + + #[test] + fn pacer_does_not_bank_credit_over_an_idle_gap() { + // After a pause longer than the interval, the next request still + // starts immediately (one slot), not a burst of pre-earned slots. + let p = Pacer::new(); + p.set_per_second(Some(100.0)); // 10ms + p.wait(); // takes a slot now + std::thread::sleep(Duration::from_millis(40)); + let start = Instant::now(); + p.wait(); // next is in the past -> immediate + assert!(start.elapsed() < Duration::from_millis(5)); + } } diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs index 22d803c..455fb27 100644 --- a/src/sync/import_exchange_ews/coordinator.rs +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -38,6 +38,7 @@ pub struct EwsImportConfig { pub mailbox_kind: MailboxKind, pub auth: EwsAuth, pub ews_connections: usize, + pub max_rps: Option, pub getitem_batch: usize, /// Byte cap for one GetItem batch; see `sync::batch`. pub getitem_batch_bytes: u64, @@ -85,6 +86,7 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result, pub top: usize, pub exception_window_years: i32, pub contact_photos: bool, @@ -88,6 +89,7 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result