From a70efdbb829ca8d7e264556f03277b1d548be782 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 12 Nov 2026 10:21:42 -0700 Subject: [PATCH] estimate: counts-only for JMAP and Microsoft Graph --estimate now covers the two remaining online sources. Both can enumerate an object's id cheaply, but a size would cost a body fetch per object, so these report counts without bytes: for each type, how many objects are new since the archive was last filled. JMAP already computes that delta in its dry run (new versus what the archive holds); the estimate just keeps the per-type summary instead of printing the dry-run table and discarding it. Graph's dry run only counted folders, so this adds a read-only item pass (src/sync/import_exchange_graph/estimate.rs): it walks the same folder tree the import does, enumerates message, event and contact ids with $select=id, and counts those not already in the archive. It skips single occurrences exactly as the import does (it stores the series master) and walks nested and default contact folders, so the count matches what an import would create and a re-estimate finds nothing new. It writes nothing. The shared new-id counter is a small, unit-tested helper. README, the --estimate help and the fallback note now distinguish sized sources from these counts-only ones. --- README.md | 6 +- src/cli.rs | 11 +- src/main.rs | 2 +- src/sync/import_exchange_graph.rs | 1 + src/sync/import_exchange_graph/coordinator.rs | 24 ++ src/sync/import_exchange_graph/estimate.rs | 237 ++++++++++++++++++ src/sync/import_exchange_graph/folders.rs | 2 +- src/sync/import_jmap.rs | 12 +- 8 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 src/sync/import_exchange_graph/estimate.rs diff --git a/README.md b/README.md index aa0bda4..34a1600 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,10 @@ takes `--dry-run` and then only reports the plan. An `import` also takes fetch — counts, bytes and a rough time — before committing, and writes nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange (`ews`) and a `takeout` (whose files it reads to count the mail, events and -contacts that are new); other sources fall back to a plain dry-run. `inspect` -reads an archive and changes nothing. +contacts that are new). For `jmap` and Microsoft Graph it reports counts only: +the enumeration gives the new-object count for free, but a size there would +cost a body fetch per object. The remaining sources fall back to a plain +dry-run. `inspect` reads an archive and changes nothing. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/src/cli.rs b/src/cli.rs index dfe3fdc..fb648a4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,9 +156,10 @@ struct GlobalArgs { help = "Preview an import: counts, sizes and a rough time, before any writes", long_help = "Preview an import before committing: for each type, how much is new since \ the archive was last filled, how many bytes that is, and a rough time.\n \ - Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \ - events and contacts) today; other sources fall back to --dry-run with a note. It \ - implies --dry-run, so it never writes. On an export it is just --dry-run." + Sizes for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, events \ + and contacts); counts only for jmap and Graph (a size there needs a body fetch per \ + object). Other sources fall back to --dry-run with a note. It implies --dry-run, so \ + it never writes. On an export it is just --dry-run." )] estimate: bool, @@ -936,6 +937,10 @@ impl Action { // EWS FindItem returns item:Size, so mail, calendars and contacts // are all counted and sized from the enumeration. Action::ImportExchangeEws(c, _) => c.estimate, + // JMAP and Graph report counts only: the dry-run enumeration gives + // the new-object count for free, but a size would cost a body fetch + // per object, which an estimate does not do. + Action::Import(c, _) | Action::ImportExchangeGraph(c, _) => c.estimate, _ => false, } } diff --git a/src/main.rs b/src/main.rs index 60f7750..f362aa8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ fn run() -> i32 { if action.estimate_requested() && !do_estimate { eprintln!( "note: --estimate isn't available for this source yet; running --dry-run instead \ - (its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS) and takeout today." + (its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS), takeout, jmap and Graph today." ); } let mut quiet_report = false; diff --git a/src/sync/import_exchange_graph.rs b/src/sync/import_exchange_graph.rs index 1d29802..c184ba2 100644 --- a/src/sync/import_exchange_graph.rs +++ b/src/sync/import_exchange_graph.rs @@ -7,6 +7,7 @@ pub mod calendar; pub mod contacts; pub mod coordinator; +pub mod estimate; pub mod files; pub mod folders; pub mod messages; diff --git a/src/sync/import_exchange_graph/coordinator.rs b/src/sync/import_exchange_graph/coordinator.rs index a163078..fdec04e 100644 --- a/src/sync/import_exchange_graph/coordinator.rs +++ b/src/sync/import_exchange_graph/coordinator.rs @@ -114,6 +114,30 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! The counts-only `--estimate` pass for a Microsoft Graph source. +//! +//! Graph can enumerate an item's id cheaply (`$select=id`), but a size would +//! cost a body fetch per item, which an estimate does not do. So this reports +//! how many mail, calendar events and contacts are new since the archive was +//! last filled -- the same id enumeration and archive diff the import runs -- +//! and no bytes. It reads folder lists and id pages only; it writes nothing. + +use std::collections::{HashMap, HashSet}; + +use rusqlite::Connection; +use serde_json::Value; + +use super::coordinator::{GraphCoordinator, enumerate_mail_folders}; +use super::folders::default_contact_folder; +use crate::db::exchange_graph_ids; +use crate::error::Error; +use crate::exchange_graph::api::{self, PREFER_TIMEZONE_UTC}; +use crate::exchange_graph::calendar_map::{EventType, classify_event_type}; +use crate::exchange_graph::types::{MailboxKind, Surfaces}; +use crate::sync::{Summary, TypeCounts}; + +/// Builds the estimate summary: a per-type row of new objects for each surface +/// the run asks for. Mirrors which surfaces the real import would touch (an +/// archive mailbox carries mail only). +pub fn run( + conn: &Connection, + ctx: &GraphCoordinator<'_>, + mailbox_kind: MailboxKind, + surfaces: &Surfaces, +) -> Result { + let mut summary = Summary::default(); + let primary = !matches!(mailbox_kind, MailboxKind::Archive); + if surfaces.mail { + summary + .per_type + .push(("email", estimate_mail(conn, ctx, mailbox_kind)?)); + } + if surfaces.calendar && primary { + summary + .per_type + .push(("calendarevent", estimate_calendar(conn, ctx)?)); + } + if surfaces.contacts && primary { + summary + .per_type + .push(("contactcard", estimate_contacts(conn, ctx)?)); + } + Ok(summary) +} + +/// Counts the mail that is new across the full folder tree. Enumerates message +/// ids folder by folder -- the same read-only tree walk a dry run does -- and +/// counts those not already in the archive, deduplicated across folders. +fn estimate_mail( + conn: &Connection, + ctx: &GraphCoordinator<'_>, + mailbox_kind: MailboxKind, +) -> Result { + let local = exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?; + let folders = enumerate_mail_folders(ctx.client, ctx.endpoints, mailbox_kind, ctx.top) + .map_err(Error::from)?; + let mut counts = TypeCounts::default(); + let mut planned: HashSet = HashSet::new(); + for folder in &folders { + let Some(fid) = folder.get("id").and_then(Value::as_str) else { + continue; + }; + let url = ctx.endpoints.folder_messages_ids(fid, ctx.top); + let stubs = match api::collect_all_values(ctx.client, &url, &[]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("estimate: mail folder {fid} enumerate: {e}")); + counts.failed += 1; + continue; + } + }; + for stub in &stubs { + count_new( + stub.get("id").and_then(Value::as_str), + &local, + &mut planned, + &mut counts, + ); + } + } + Ok(counts) +} + +/// Counts the calendar events that are new. Enumerates event ids per calendar +/// and skips single `Occurrence` items exactly as the import does -- it stores +/// the series master, not each occurrence -- so a re-estimate finds nothing +/// new once imported. +fn estimate_calendar(conn: &Connection, ctx: &GraphCoordinator<'_>) -> Result { + let local = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?; + let calendars = api::collect_all_values(ctx.client, &ctx.endpoints.calendars(ctx.top), &[]) + .map_err(Error::from)?; + let mut counts = TypeCounts::default(); + let mut planned: HashSet = HashSet::new(); + for cal in &calendars { + let Some(cid) = cal.get("id").and_then(Value::as_str) else { + continue; + }; + let url = ctx.endpoints.calendar_events_ids(cid, ctx.top); + let stubs = match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("estimate: calendar {cid} enumerate: {e}")); + counts.failed += 1; + continue; + } + }; + for stub in &stubs { + if matches!(classify_event_type(stub), EventType::Occurrence) { + continue; + } + count_new( + stub.get("id").and_then(Value::as_str), + &local, + &mut planned, + &mut counts, + ); + } + } + Ok(counts) +} + +/// Counts the contacts that are new across every address book, including nested +/// folders and the default folder, which is the same set the import walks. +fn estimate_contacts(conn: &Connection, ctx: &GraphCoordinator<'_>) -> Result { + let local = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?; + let books = discover_contact_folder_ids(ctx)?; + let mut counts = TypeCounts::default(); + let mut planned: HashSet = HashSet::new(); + for bid in &books { + let url = ctx.endpoints.contact_folder_contacts_ids(bid, ctx.top); + let ids = match api::collect_all_ids(ctx.client, &url, &[]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("estimate: contact folder {bid} enumerate: {e}")); + counts.failed += 1; + continue; + } + }; + for id in &ids { + count_new(Some(id.as_str()), &local, &mut planned, &mut counts); + } + } + Ok(counts) +} + +/// The read-only address-book discovery the import does: the listed folders, +/// the default folder (which may not be listed), and every nested child. +fn discover_contact_folder_ids(ctx: &GraphCoordinator<'_>) -> Result, Error> { + let server = api::collect_all_values(ctx.client, &ctx.endpoints.contact_folders(ctx.top), &[]) + .map_err(Error::from)?; + let mut seen: HashSet = server + .iter() + .filter_map(|f| f.get("id").and_then(Value::as_str)) + .map(str::to_owned) + .collect(); + if let Some(default) = default_contact_folder(ctx) + && let Some(id) = default.get("id").and_then(Value::as_str) + { + seen.insert(id.to_owned()); + } + let mut frontier: Vec = seen.iter().cloned().collect(); + while let Some(parent) = frontier.pop() { + let url = ctx.endpoints.contact_folder_children(&parent, ctx.top); + let children = api::collect_all_values(ctx.client, &url, &[]).map_err(Error::from)?; + for child in children { + if let Some(id) = child.get("id").and_then(Value::as_str) + && seen.insert(id.to_owned()) + { + frontier.push(id.to_owned()); + } + } + } + Ok(seen.into_iter().collect()) +} + +/// Counts an enumerated id as new when it is neither already in the archive nor +/// already seen in this pass (the same id can appear in two folders). +fn count_new( + id: Option<&str>, + local: &HashMap, + planned: &mut HashSet, + counts: &mut TypeCounts, +) { + let Some(id) = id else { return }; + if local.contains_key(id) { + return; + } + if planned.insert(id.to_owned()) { + counts.created += 1; + counts.fetched += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn count_new_skips_archived_and_dedups_within_a_pass() { + let mut local: HashMap = HashMap::new(); + local.insert("archived".to_owned(), 1); + let mut planned: HashSet = HashSet::new(); + let mut counts = TypeCounts::default(); + + // Already in the archive: not new. + count_new(Some("archived"), &local, &mut planned, &mut counts); + // New: counted once... + count_new(Some("A"), &local, &mut planned, &mut counts); + // ...and the same id in another folder does not count again. + count_new(Some("A"), &local, &mut planned, &mut counts); + // Another new one. + count_new(Some("B"), &local, &mut planned, &mut counts); + // A stub with no id is ignored. + count_new(None, &local, &mut planned, &mut counts); + + assert_eq!(counts.fetched, 2, "A and B are new; A counts once"); + assert_eq!(counts.created, 2); + assert_eq!(counts.bytes, 0, "counts-only: no sizes"); + } +} diff --git a/src/sync/import_exchange_graph/folders.rs b/src/sync/import_exchange_graph/folders.rs index e9f6dce..cce6419 100644 --- a/src/sync/import_exchange_graph/folders.rs +++ b/src/sync/import_exchange_graph/folders.rs @@ -341,7 +341,7 @@ pub fn reconcile_address_books( Ok(out) } -fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option { +pub(crate) fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option { match ctx.client.get_json(&ctx.endpoints.default_contact_folder()) { Ok(value) if value.get("id").and_then(Value::as_str).is_some() => return Some(value), Ok(_) => {} diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs index b3a7ae9..abf434d 100644 --- a/src/sync/import_jmap.rs +++ b/src/sync/import_jmap.rs @@ -298,8 +298,12 @@ fn run_into( } if ctx.dry_run() { - print_dry_run(&dry_rows); - *summary = Summary::default(); + // A plain dry run prints its own table and discards the counts; an + // estimate keeps the per-type summary so the caller can render it. + if !ctx.estimate() { + print_dry_run(&dry_rows); + *summary = Summary::default(); + } return Ok(()); } @@ -399,6 +403,10 @@ fn reconcile_type( d.vanished.len() as u64, d.present.len() as u64, )); + // --estimate keeps the summary (counts only; JMAP would need a body + // fetch per object to size it, which an estimate does not do), so the + // new-object count has to land in the per-type row the caller pushes. + counts.fetched = d.new.len() as u64; return Ok(()); } -- 2.54.0