diff --git a/README.md b/README.md index db5fa50..58b3a00 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ later `import` into the same archive adds what arrived since. Every command takes `--dry-run` and then only reports the plan. An `import` also takes `--estimate`, which connects to the source and reports how much is new to fetch — counts, bytes and a rough time — before committing, and writes -nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav` and a -`takeout`'s mail (which it reads to count); other sources fall back to a plain -dry-run. `inspect` reads an archive and changes nothing. +nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange +(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall +back to a plain dry-run. `inspect` reads an archive and changes nothing. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/src/cli.rs b/src/cli.rs index 7c5ac8f..c7c91ab 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,9 +156,9 @@ struct GlobalArgs { help = "Preview an import: counts, sizes and a rough time, before any writes", long_help = "Preview an import before committing: for each type, how much is new since \ the archive was last filled, how many bytes that is, and a rough time.\n \ - Available for imap, maildir, caldav, carddav and takeout (its mail) today; other \ - sources fall back to --dry-run with a note. It implies --dry-run, so it never writes. \ - On an export it is just --dry-run." + Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \ + today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \ + never writes. On an export it is just --dry-run." )] estimate: bool, @@ -933,6 +933,9 @@ impl Action { // Takeout reads the mbox files to count mail; its calendars and // contacts are not itemized yet. Action::ImportTakeout(c, _) => c.estimate, + // EWS FindItem returns item:Size, so mail, calendars and contacts + // are all counted and sized from the enumeration. + Action::ImportExchangeEws(c, _) => c.estimate, _ => false, } } diff --git a/src/main.rs b/src/main.rs index bc2c2ba..60f7750 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ fn run() -> i32 { if action.estimate_requested() && !do_estimate { eprintln!( "note: --estimate isn't available for this source yet; running --dry-run instead \ - (its plan is below). Estimates work for imap, maildir, caldav, carddav and takeout today." + (its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS) and takeout today." ); } let mut quiet_report = false; diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs index eeabf1d..22d803c 100644 --- a/src/sync/import_exchange_ews/coordinator.rs +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -19,7 +19,9 @@ use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS}; use crate::sync::{CommonConfig, Summary, TypeCounts}; use super::folders::{self, plan_folders}; +use super::items::{self, ItemRunCtx}; use super::{calendar, contacts, messages}; +use crate::db::exchange_ews_ids; use crate::net::CertOverride; #[derive(Debug, Clone)] @@ -102,7 +104,16 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result Result Result { let plan = plan_folders( @@ -218,9 +231,68 @@ fn run_dry( summary.per_type.push(("mailbox", mailbox_counts)); summary.per_type.push(("calendar", calendar_counts)); summary.per_type.push(("addressbook", addressbook_counts)); + + // --estimate: enumerate each folder's items (FindItem gives item:Size, so + // this never fetches a body) and count and size what is new against the + // archive. A plain dry run stops at the folder counts above. + if want_sizes { + let ctx = ItemRunCtx { + client, + url: session_url, + source_id, + batch_size: config.getitem_batch.max(1), + batch_bytes: config.getitem_batch_bytes.max(1), + attachment_batch: config.attachment_batch.max(1), + connections: config.ews_connections.clamp(1, 8), + // FindItem (not SyncFolderItems) so every enumeration carries sizes. + use_syncfolderitems: false, + sync_batch: items::SYNC_BATCH_MAX, + logger, + }; + let email = estimate_items(conn, &ctx, &plan.mail, exchange_ews_ids::EMAIL); + let events = estimate_items(conn, &ctx, &plan.calendar, exchange_ews_ids::CALENDAR_EVENT); + let cards = estimate_items(conn, &ctx, &plan.contacts, exchange_ews_ids::CONTACT_CARD); + summary.per_type.push(("email", email)); + summary.per_type.push(("calendarevent", events)); + summary.per_type.push(("contactcard", cards)); + } Ok(summary) } +/// Counts and sizes the items in `folders` that are new against the archive, +/// for `--estimate`. FindItem returns `item:Size`, so this reads no bodies and +/// writes nothing; the diff is the same one the import uses, so a re-estimate +/// against a part-filled archive reports only what is left. +fn estimate_items( + conn: &Connection, + ctx: &ItemRunCtx<'_>, + folders: &[folders::ClassifiedFolder], + item_type: &str, +) -> TypeCounts { + let mut counts = TypeCounts::default(); + for cf in folders { + let fid = &cf.folder.folder_id; + let outcome = match items::enumerate_folder(ctx, fid, None) { + Ok(o) => o, + Err(e) => { + ctx.logger.warn(&format!( + "dry-run enumerate {:?}: {e}", + cf.folder.display_name + )); + counts.failed += 1; + continue; + } + }; + let local = exchange_ews_ids::items_in_folder(conn, ctx.source_id, item_type, &fid.id) + .unwrap_or_default(); + let (new, bytes) = items::new_count_and_bytes(&outcome, &local); + counts.created += new; + counts.fetched += new; + counts.bytes += bytes; + } + counts +} + fn resolve_auth(auth: &EwsAuth) -> Result<(Auth, Option), Error> { match auth { EwsAuth::Basic { user, password } => Ok(( diff --git a/src/sync/import_exchange_ews/items.rs b/src/sync/import_exchange_ews/items.rs index 1d323e5..39b3f2c 100644 --- a/src/sync/import_exchange_ews/items.rs +++ b/src/sync/import_exchange_ews/items.rs @@ -221,6 +221,25 @@ pub fn plan_for( } } +/// Count the new items an enumeration turned up and sum the sizes the server +/// reported for them. Sizes come from FindItem's `item:Size`; an item the +/// server returned without a size still counts, it just adds no bytes. Used by +/// `--estimate`, which enumerates with FindItem precisely so the sizes are +/// present. +pub fn new_count_and_bytes( + outcome: &EnumerationOutcome, + local: &[crate::db::exchange_ews_ids::ItemRow], +) -> (u64, u64) { + let plan = plan_for(outcome, local); + let count = plan.new.len() as u64; + let bytes = plan + .new + .iter() + .filter_map(|id| outcome.sizes.get(&id.id).copied()) + .sum(); + (count, bytes) +} + fn diff_delta( server_changes: &[EnumeratedItem], deletions: &[String], diff --git a/tests/mock_exchange_ews.rs b/tests/mock_exchange_ews.rs index 539963c..5362d5e 100644 --- a/tests/mock_exchange_ews.rs +++ b/tests/mock_exchange_ews.rs @@ -534,6 +534,74 @@ fn coordinator_diff_classifies_new_vanished_changed_unchanged() { assert_eq!(plan.vanished[0].0, "Z"); } +#[test] +fn estimate_counts_new_items_and_sums_their_sizes() { + use inbuxa_migrate::db::exchange_ews_ids::ItemRow; + use inbuxa_migrate::sync::import_exchange_ews::items::{ + EnumeratedItem, EnumerationMode, EnumerationOutcome, new_count_and_bytes, + }; + use std::collections::HashMap; + + // Three items on the server; A is already local (not new), B and C are new. + let items = vec![ + EnumeratedItem { + element: "Message".into(), + id: ItemId::new("A", "ck-1"), + }, + EnumeratedItem { + element: "Message".into(), + id: ItemId::new("B", "ck-1"), + }, + EnumeratedItem { + element: "Message".into(), + id: ItemId::new("C", "ck-1"), + }, + ]; + let mut sizes = HashMap::new(); + sizes.insert("A".to_string(), 1000u64); + sizes.insert("B".to_string(), 2048u64); + // C has no reported size: it still counts, but adds no bytes. + let outcome = EnumerationOutcome { + items, + mode: EnumerationMode::Full, + sizes, + }; + let local = vec![ItemRow { + item_id: "A".into(), + change_key: "ck-1".into(), + local_id: 1, + }]; + + let (count, bytes) = new_count_and_bytes(&outcome, &local); + assert_eq!(count, 2, "B and C are new; A is already local"); + assert_eq!( + bytes, 2048, + "only B's size is known; C has none; A is not new" + ); + + // Nothing new once everything is local: no count, no bytes. + let all_local = vec![ + ItemRow { + item_id: "A".into(), + change_key: "ck-1".into(), + local_id: 1, + }, + ItemRow { + item_id: "B".into(), + change_key: "ck-1".into(), + local_id: 2, + }, + ItemRow { + item_id: "C".into(), + change_key: "ck-1".into(), + local_id: 3, + }, + ]; + let (count, bytes) = new_count_and_bytes(&outcome, &all_local); + assert_eq!(count, 0); + assert_eq!(bytes, 0); +} + #[test] fn source_change_protection_refuses_different_account() { use inbuxa_migrate::db;