diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eaf4ce..a584bcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a `INBUXA_MIGRATE_API_KEY_FILE`, and so on). One trailing newline is ignored; setting both a variable and its `_FILE` form is an error. This keeps a secret out of both the environment and the command line. +- `import exchange-graph` and `import exchange-ews` take `--max-rps` to cap + how many requests start per second across all worker connections -- a + proactive throttle to stay under a rate-limited Microsoft 365 tenant's + limit, alongside the existing honoring of 429/`Retry-After`. Off by + default (uncapped). ## [27.1-rc.12] diff --git a/docs/usage.md b/docs/usage.md index d247dcf..6f7421d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -266,7 +266,8 @@ inbuxa-migrate import exchange-ews \ (--auth-basic [--auth-password ] \ | --auth-bearer [TOKEN] [--ews-tenant --ews-client-id \ (--ews-device-code | --ews-client-secret )]) \ - [--ews-connections <1..8>] [--ews-getitem-batch ] [--ews-getitem-batch-mib ] \ + [--ews-connections <1..8>] [--max-rps ] \ + [--ews-getitem-batch ] [--ews-getitem-batch-mib ] \ [--ews-attachment-batch ] \ [--ews-no-syncfolderitems] \ @@ -283,6 +284,11 @@ time. The byte cap applies where Exchange reports each item's size, which it does on a full listing; items found through an incremental sync are batched by count. +The server throttles on its own, and a 429 or 503 is honored (its +`Retry-After`, then a shared backoff across the workers). To stay under a +known limit in the first place, `--max-rps` caps how many requests start per +second across all connections; leave it off to run uncapped. + For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from October 1, 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on April 1, 2027 it is switched off for every tenant. On-premises Exchange Server is not affected. ### Microsoft Exchange (Graph) @@ -294,13 +300,18 @@ inbuxa-migrate import exchange-graph \ [--mailbox-kind primary|archive] \ [--objects mail,calendar,contacts] \ [--event-body-format text|html] \ - [--graph-connections <1..16>] [--top <1..1000>] \ + [--graph-connections <1..16>] [--max-rps ] [--top <1..1000>] \ ``` Imports a mailbox from Exchange Online through Microsoft Graph. Without `--access-token` it signs in with the interactive device-code flow. `public-folders` is rejected here: Graph does not expose public folders, so they can only be imported with `exchange-ews`, which for Exchange Online is subject to the retirement described above. +Graph throttles per app and per mailbox, and a 429 or 503 is honored (its +`Retry-After`, then a shared backoff across the workers). `--max-rps` caps how +many requests start per second across all connections, to stay under a tenant's +limit before it pushes back; leave it off to run uncapped. + With `--app-only` the app signs in as itself, with its client secret, and reads the mailbox `--user` names. No one signs in, so a whole tenant moves with one app, in a [plan](#migrate-many-accounts) with `--user {email}`. diff --git a/src/cli.rs b/src/cli.rs index ed8c4f5..81f289e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1820,6 +1820,14 @@ pub struct ExchangeEwsImportArgs { )] ews_connections: usize, + #[arg( + long, + value_name = "RPS", + value_parser = parse_max_rps, + help = "Cap requests per second across all connections (proactive throttle for a rate-limited server)" + )] + max_rps: Option, + #[arg( long, value_name = "N", @@ -1884,6 +1892,7 @@ fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result, + #[arg( long, value_name = "N", @@ -2103,6 +2120,14 @@ fn parse_event_body_format(s: &str) -> Result { EventBodyFormat::parse(s).map_err(|e| e.to_string()) } +/// A requests-per-second cap: a positive, finite number. +fn parse_max_rps(s: &str) -> Result { + match s.parse::() { + Ok(v) if v.is_finite() && v > 0.0 => Ok(v), + _ => Err("expected a positive number of requests per second".to_owned()), + } +} + fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result { let common = common_config(&args.global, args.archive.clone()); let mailbox_kind = args.mailbox_kind; @@ -2134,6 +2159,7 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result EwsImportConfig { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "exchange-ews", + "--url", + "https://ews.example.com/EWS/Exchange.asmx", + "--auth-bearer", + "tok", + ]; + argv.extend_from_slice(extra); + argv.push("archive.sqlite"); + let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected: {e}")); + match cli.resolve().unwrap() { + Action::ImportExchangeEws(_, c) => c, + _ => panic!("expected ImportExchangeEws"), + } + }; + assert_eq!(config(&[]).max_rps, None); + assert_eq!(config(&["--max-rps", "4"]).max_rps, Some(4.0)); + } + #[test] fn parse_duration_reads_units_and_rejects_nonsense() { use std::time::Duration; diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs index 0d64d44..01f83e9 100644 --- a/src/exchange_ews/client.rs +++ b/src/exchange_ews/client.rs @@ -19,7 +19,7 @@ use crate::exchange_ews::retry::{FaultDisposition, classify_fault, classify_http use crate::exchange_ews::soap::{EnvelopeOptions, soap_action, wrap_envelope}; use crate::exchange_ews::types::ServerVersion; use crate::jmap::http::{Auth, RetryPolicy, retry_after_header}; -use crate::jmap::retry::{self, Disposition, RateLimitState}; +use crate::jmap::retry::{self, Disposition, Pacer, RateLimitState}; use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::net::{CertOverride, tls, with_timeouts}; @@ -36,6 +36,7 @@ struct Inner { affinity_cookie: Mutex>, retry: RetryPolicy, rate_limit: RateLimitState, + pace: Pacer, version: Mutex, log_level: AtomicU8, retries_total: AtomicU64, @@ -90,6 +91,7 @@ impl EwsClient { affinity_cookie: Mutex::new(None), retry, rate_limit: RateLimitState::new(), + pace: Pacer::new(), version: Mutex::new(ServerVersion::Exchange2013Sp1), log_level: AtomicU8::new(LEVEL_DEFAULT), retries_total: AtomicU64::new(0), @@ -124,6 +126,12 @@ impl EwsClient { .store(logger.level(), Ordering::Relaxed); } + /// Cap how often a request may start, in requests per second, across all + /// worker connections. `None` or a non-positive rate leaves it uncapped. + pub fn set_max_rps(&self, max_rps: Option) { + self.inner.pace.set_per_second(max_rps); + } + pub fn logger(&self) -> Logger { Logger::new(self.inner.log_level.load(Ordering::Relaxed)) } @@ -209,6 +217,7 @@ impl EwsClient { let mut attempt: u32 = 0; loop { self.inner.rate_limit.cooldown().wait(); + self.inner.pace.wait(); let sent_version = self.server_version(); let envelope = self.wrap(body); let outcome = self.one_attempt(url, &envelope, action); diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs index 14c13de..1123f82 100644 --- a/src/exchange_graph/client.rs +++ b/src/exchange_graph/client.rs @@ -18,7 +18,7 @@ use ureq::{ResponseExt, http::Uri}; use crate::exchange_graph::error::GraphError; use crate::exchange_graph::retry::{HttpClass, classify_http_status, is_throttled}; use crate::jmap::http::{RetryPolicy, cross_host, retry_after_header}; -use crate::jmap::retry::{self, RateLimitState}; +use crate::jmap::retry::{self, Pacer, RateLimitState}; use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::net::{CertOverride, tls, with_timeouts}; @@ -68,6 +68,7 @@ struct Inner { bearer: Mutex, retry: RetryPolicy, rate_limit: RateLimitState, + pace: Pacer, log_level: AtomicU8, retries_total: AtomicU64, retry_after_sleeps: AtomicU64, @@ -122,6 +123,7 @@ impl GraphClient { bearer: Mutex::new(bearer), retry, rate_limit: RateLimitState::new(), + pace: Pacer::new(), log_level: AtomicU8::new(LEVEL_DEFAULT), retries_total: AtomicU64::new(0), retry_after_sleeps: AtomicU64::new(0), @@ -145,6 +147,12 @@ impl GraphClient { .store(logger.level(), Ordering::Relaxed); } + /// Cap how often a request may start, in requests per second, across all + /// worker connections. `None` or a non-positive rate leaves it uncapped. + pub fn set_max_rps(&self, max_rps: Option) { + self.inner.pace.set_per_second(max_rps); + } + fn logger(&self) -> Logger { Logger::new(self.inner.log_level.load(Ordering::Relaxed)) } @@ -210,6 +218,7 @@ impl GraphClient { let mut attempt: u32 = 0; loop { self.inner.rate_limit.cooldown().wait(); + self.inner.pace.wait(); self.inner.requests_total.fetch_add(1, Ordering::Relaxed); let outcome = self.one_attempt(method, url, accept, extra_prefer); match outcome { diff --git a/src/jmap/retry.rs b/src/jmap/retry.rs index 3ea86cd..f988e44 100644 --- a/src/jmap/retry.rs +++ b/src/jmap/retry.rs @@ -6,7 +6,7 @@ use std::cell::Cell; use std::sync::Mutex; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::jmap::http::RetryPolicy; @@ -178,6 +178,69 @@ impl RateLimitState { } } +/// A proactive request-rate cap, shared across worker threads. It spaces the +/// *start* of successive requests at least one interval apart, so a known +/// provider limit (Microsoft Graph, Google) is kept to in the first place, +/// rather than reactively, after the server returns 429 and the [`Cooldown`] +/// engages. The interval is set once, after the client is built (like a +/// logger), and a zero interval disables it so there is no cost on the hot +/// path when no cap is asked for. Requests in flight are not limited — only +/// how often a new one may begin — so it composes with the connection count. +#[derive(Debug, Default)] +pub struct Pacer { + /// Minimum spacing between request starts, in nanoseconds; 0 disables. + min_interval_nanos: AtomicU64, + next: Mutex>, +} + +impl Pacer { + pub fn new() -> Pacer { + Pacer { + min_interval_nanos: AtomicU64::new(0), + next: Mutex::new(None), + } + } + + /// Set the cap from a requests-per-second figure. `None`, zero, or a + /// non-finite rate disables pacing. A later call replaces the cap. + pub fn set_per_second(&self, max_rps: Option) { + let nanos = match max_rps { + Some(rps) if rps.is_finite() && rps > 0.0 => { + (1.0e9 / rps).round().clamp(1.0, u64::MAX as f64) as u64 + } + _ => 0, + }; + self.min_interval_nanos.store(nanos, Ordering::Relaxed); + } + + pub fn is_enabled(&self) -> bool { + self.min_interval_nanos.load(Ordering::Relaxed) != 0 + } + + /// Block until this thread may start its request, then reserve its slot. + /// Holding the lock across the sleep is deliberate: it serializes the + /// reservations, so N waiters leave one interval apart instead of all at + /// once. An idle gap does not bank credit — the next request starts now, + /// never a burst. + pub fn wait(&self) { + let nanos = self.min_interval_nanos.load(Ordering::Relaxed); + if nanos == 0 { + return; + } + let interval = Duration::from_nanos(nanos); + let mut next = self.next.lock().unwrap_or_else(|p| p.into_inner()); + let now = Instant::now(); + let start = match *next { + Some(t) if t > now => { + std::thread::sleep(t - now); + t + } + _ => now, + }; + *next = Some(start + interval); + } +} + pub fn throttle_backoff(policy: &RetryPolicy, level: u32) -> Duration { let shift = level.saturating_sub(1).min(20); let scaled = THROTTLE_BASE @@ -340,4 +403,61 @@ mod tests { assert_eq!(d, Duration::from_millis(750)); assert_eq!(s.level(), 1); } + + #[test] + fn pacer_disabled_by_default_and_by_zero_or_none() { + let p = Pacer::new(); + assert!(!p.is_enabled()); + p.set_per_second(Some(10.0)); + assert!(p.is_enabled()); + p.set_per_second(None); + assert!(!p.is_enabled()); + p.set_per_second(Some(0.0)); + assert!(!p.is_enabled()); + p.set_per_second(Some(-5.0)); + assert!(!p.is_enabled()); + p.set_per_second(Some(f64::NAN)); + assert!(!p.is_enabled()); + } + + #[test] + fn pacer_spaces_successive_waits_by_the_interval() { + // 50 rps -> 20ms apart. The first wait returns at once (no prior + // slot); the next few are each spaced ~one interval on. + let p = Pacer::new(); + p.set_per_second(Some(50.0)); + let start = Instant::now(); + for _ in 0..4 { + p.wait(); + } + // Three gaps of 20ms = 60ms minimum after the immediate first. + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(55), + "four 50rps waits took {elapsed:?}, expected >= ~60ms" + ); + } + + #[test] + fn pacer_wait_is_immediate_when_disabled() { + let p = Pacer::new(); + let start = Instant::now(); + for _ in 0..1000 { + p.wait(); + } + assert!(start.elapsed() < Duration::from_millis(50)); + } + + #[test] + fn pacer_does_not_bank_credit_over_an_idle_gap() { + // After a pause longer than the interval, the next request still + // starts immediately (one slot), not a burst of pre-earned slots. + let p = Pacer::new(); + p.set_per_second(Some(100.0)); // 10ms + p.wait(); // takes a slot now + std::thread::sleep(Duration::from_millis(40)); + let start = Instant::now(); + p.wait(); // next is in the past -> immediate + assert!(start.elapsed() < Duration::from_millis(5)); + } } diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs index 22d803c..455fb27 100644 --- a/src/sync/import_exchange_ews/coordinator.rs +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -38,6 +38,7 @@ pub struct EwsImportConfig { pub mailbox_kind: MailboxKind, pub auth: EwsAuth, pub ews_connections: usize, + pub max_rps: Option, pub getitem_batch: usize, /// Byte cap for one GetItem batch; see `sync::batch`. pub getitem_batch_bytes: u64, @@ -85,6 +86,7 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result, pub top: usize, pub exception_window_years: i32, pub contact_photos: bool, @@ -88,6 +89,7 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result