diff --git a/README.md b/README.md index aa0bda4..34a1600 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,10 @@ takes `--dry-run` and then only reports the plan. An `import` also takes fetch — counts, bytes and a rough time — before committing, and writes nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange (`ews`) and a `takeout` (whose files it reads to count the mail, events and -contacts that are new); other sources fall back to a plain dry-run. `inspect` -reads an archive and changes nothing. +contacts that are new). For `jmap` and Microsoft Graph it reports counts only: +the enumeration gives the new-object count for free, but a size there would +cost a body fetch per object. The remaining sources fall back to a plain +dry-run. `inspect` reads an archive and changes nothing. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/src/cli.rs b/src/cli.rs index dfe3fdc..fb648a4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,9 +156,10 @@ struct GlobalArgs { help = "Preview an import: counts, sizes and a rough time, before any writes", long_help = "Preview an import before committing: for each type, how much is new since \ the archive was last filled, how many bytes that is, and a rough time.\n \ - Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \ - events and contacts) today; other sources fall back to --dry-run with a note. It \ - implies --dry-run, so it never writes. On an export it is just --dry-run." + Sizes for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, events \ + and contacts); counts only for jmap and Graph (a size there needs a body fetch per \ + object). Other sources fall back to --dry-run with a note. It implies --dry-run, so \ + it never writes. On an export it is just --dry-run." )] estimate: bool, @@ -936,6 +937,10 @@ impl Action { // EWS FindItem returns item:Size, so mail, calendars and contacts // are all counted and sized from the enumeration. Action::ImportExchangeEws(c, _) => c.estimate, + // JMAP and Graph report counts only: the dry-run enumeration gives + // the new-object count for free, but a size would cost a body fetch + // per object, which an estimate does not do. + Action::Import(c, _) | Action::ImportExchangeGraph(c, _) => c.estimate, _ => false, } } diff --git a/src/main.rs b/src/main.rs index 60f7750..f362aa8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ fn run() -> i32 { if action.estimate_requested() && !do_estimate { eprintln!( "note: --estimate isn't available for this source yet; running --dry-run instead \ - (its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS) and takeout today." + (its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS), takeout, jmap and Graph today." ); } let mut quiet_report = false; diff --git a/src/sync/import_exchange_graph.rs b/src/sync/import_exchange_graph.rs index 1d29802..c184ba2 100644 --- a/src/sync/import_exchange_graph.rs +++ b/src/sync/import_exchange_graph.rs @@ -7,6 +7,7 @@ pub mod calendar; pub mod contacts; pub mod coordinator; +pub mod estimate; pub mod files; pub mod folders; pub mod messages; diff --git a/src/sync/import_exchange_graph/coordinator.rs b/src/sync/import_exchange_graph/coordinator.rs index a163078..fdec04e 100644 --- a/src/sync/import_exchange_graph/coordinator.rs +++ b/src/sync/import_exchange_graph/coordinator.rs @@ -114,6 +114,30 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! The counts-only `--estimate` pass for a Microsoft Graph source. +//! +//! Graph can enumerate an item's id cheaply (`$select=id`), but a size would +//! cost a body fetch per item, which an estimate does not do. So this reports +//! how many mail, calendar events and contacts are new since the archive was +//! last filled -- the same id enumeration and archive diff the import runs -- +//! and no bytes. It reads folder lists and id pages only; it writes nothing. + +use std::collections::{HashMap, HashSet}; + +use rusqlite::Connection; +use serde_json::Value; + +use super::coordinator::{GraphCoordinator, enumerate_mail_folders}; +use super::folders::default_contact_folder; +use crate::db::exchange_graph_ids; +use crate::error::Error; +use crate::exchange_graph::api::{self, PREFER_TIMEZONE_UTC}; +use crate::exchange_graph::calendar_map::{EventType, classify_event_type}; +use crate::exchange_graph::types::{MailboxKind, Surfaces}; +use crate::sync::{Summary, TypeCounts}; + +/// Builds the estimate summary: a per-type row of new objects for each surface +/// the run asks for. Mirrors which surfaces the real import would touch (an +/// archive mailbox carries mail only). +pub fn run( + conn: &Connection, + ctx: &GraphCoordinator<'_>, + mailbox_kind: MailboxKind, + surfaces: &Surfaces, +) -> Result { + let mut summary = Summary::default(); + let primary = !matches!(mailbox_kind, MailboxKind::Archive); + if surfaces.mail { + summary + .per_type + .push(("email", estimate_mail(conn, ctx, mailbox_kind)?)); + } + if surfaces.calendar && primary { + summary + .per_type + .push(("calendarevent", estimate_calendar(conn, ctx)?)); + } + if surfaces.contacts && primary { + summary + .per_type + .push(("contactcard", estimate_contacts(conn, ctx)?)); + } + Ok(summary) +} + +/// Counts the mail that is new across the full folder tree. Enumerates message +/// ids folder by folder -- the same read-only tree walk a dry run does -- and +/// counts those not already in the archive, deduplicated across folders. +fn estimate_mail( + conn: &Connection, + ctx: &GraphCoordinator<'_>, + mailbox_kind: MailboxKind, +) -> Result { + let local = exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?; + let folders = enumerate_mail_folders(ctx.client, ctx.endpoints, mailbox_kind, ctx.top) + .map_err(Error::from)?; + let mut counts = TypeCounts::default(); + let mut planned: HashSet = HashSet::new(); + for folder in &folders { + let Some(fid) = folder.get("id").and_then(Value::as_str) else { + continue; + }; + let url = ctx.endpoints.folder_messages_ids(fid, ctx.top); + let stubs = match api::collect_all_values(ctx.client, &url, &[]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("estimate: mail folder {fid} enumerate: {e}")); + counts.failed += 1; + continue; + } + }; + for stub in &stubs { + count_new( + stub.get("id").and_then(Value::as_str), + &local, + &mut planned, + &mut counts, + ); + } + } + Ok(counts) +} + +/// Counts the calendar events that are new. Enumerates event ids per calendar +/// and skips single `Occurrence` items exactly as the import does -- it stores +/// the series master, not each occurrence -- so a re-estimate finds nothing +/// new once imported. +fn estimate_calendar(conn: &Connection, ctx: &GraphCoordinator<'_>) -> Result { + let local = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?; + let calendars = api::collect_all_values(ctx.client, &ctx.endpoints.calendars(ctx.top), &[]) + .map_err(Error::from)?; + let mut counts = TypeCounts::default(); + let mut planned: HashSet = HashSet::new(); + for cal in &calendars { + let Some(cid) = cal.get("id").and_then(Value::as_str) else { + continue; + }; + let url = ctx.endpoints.calendar_events_ids(cid, ctx.top); + let stubs = match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("estimate: calendar {cid} enumerate: {e}")); + counts.failed += 1; + continue; + } + }; + for stub in &stubs { + if matches!(classify_event_type(stub), EventType::Occurrence) { + continue; + } + count_new( + stub.get("id").and_then(Value::as_str), + &local, + &mut planned, + &mut counts, + ); + } + } + Ok(counts) +} + +/// Counts the contacts that are new across every address book, including nested +/// folders and the default folder, which is the same set the import walks. +fn estimate_contacts(conn: &Connection, ctx: &GraphCoordinator<'_>) -> Result { + let local = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?; + let books = discover_contact_folder_ids(ctx)?; + let mut counts = TypeCounts::default(); + let mut planned: HashSet = HashSet::new(); + for bid in &books { + let url = ctx.endpoints.contact_folder_contacts_ids(bid, ctx.top); + let ids = match api::collect_all_ids(ctx.client, &url, &[]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("estimate: contact folder {bid} enumerate: {e}")); + counts.failed += 1; + continue; + } + }; + for id in &ids { + count_new(Some(id.as_str()), &local, &mut planned, &mut counts); + } + } + Ok(counts) +} + +/// The read-only address-book discovery the import does: the listed folders, +/// the default folder (which may not be listed), and every nested child. +fn discover_contact_folder_ids(ctx: &GraphCoordinator<'_>) -> Result, Error> { + let server = api::collect_all_values(ctx.client, &ctx.endpoints.contact_folders(ctx.top), &[]) + .map_err(Error::from)?; + let mut seen: HashSet = server + .iter() + .filter_map(|f| f.get("id").and_then(Value::as_str)) + .map(str::to_owned) + .collect(); + if let Some(default) = default_contact_folder(ctx) + && let Some(id) = default.get("id").and_then(Value::as_str) + { + seen.insert(id.to_owned()); + } + let mut frontier: Vec = seen.iter().cloned().collect(); + while let Some(parent) = frontier.pop() { + let url = ctx.endpoints.contact_folder_children(&parent, ctx.top); + let children = api::collect_all_values(ctx.client, &url, &[]).map_err(Error::from)?; + for child in children { + if let Some(id) = child.get("id").and_then(Value::as_str) + && seen.insert(id.to_owned()) + { + frontier.push(id.to_owned()); + } + } + } + Ok(seen.into_iter().collect()) +} + +/// Counts an enumerated id as new when it is neither already in the archive nor +/// already seen in this pass (the same id can appear in two folders). +fn count_new( + id: Option<&str>, + local: &HashMap, + planned: &mut HashSet, + counts: &mut TypeCounts, +) { + let Some(id) = id else { return }; + if local.contains_key(id) { + return; + } + if planned.insert(id.to_owned()) { + counts.created += 1; + counts.fetched += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn count_new_skips_archived_and_dedups_within_a_pass() { + let mut local: HashMap = HashMap::new(); + local.insert("archived".to_owned(), 1); + let mut planned: HashSet = HashSet::new(); + let mut counts = TypeCounts::default(); + + // Already in the archive: not new. + count_new(Some("archived"), &local, &mut planned, &mut counts); + // New: counted once... + count_new(Some("A"), &local, &mut planned, &mut counts); + // ...and the same id in another folder does not count again. + count_new(Some("A"), &local, &mut planned, &mut counts); + // Another new one. + count_new(Some("B"), &local, &mut planned, &mut counts); + // A stub with no id is ignored. + count_new(None, &local, &mut planned, &mut counts); + + assert_eq!(counts.fetched, 2, "A and B are new; A counts once"); + assert_eq!(counts.created, 2); + assert_eq!(counts.bytes, 0, "counts-only: no sizes"); + } +} diff --git a/src/sync/import_exchange_graph/folders.rs b/src/sync/import_exchange_graph/folders.rs index e9f6dce..cce6419 100644 --- a/src/sync/import_exchange_graph/folders.rs +++ b/src/sync/import_exchange_graph/folders.rs @@ -341,7 +341,7 @@ pub fn reconcile_address_books( Ok(out) } -fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option { +pub(crate) fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option { match ctx.client.get_json(&ctx.endpoints.default_contact_folder()) { Ok(value) if value.get("id").and_then(Value::as_str).is_some() => return Some(value), Ok(_) => {} diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs index b3a7ae9..abf434d 100644 --- a/src/sync/import_jmap.rs +++ b/src/sync/import_jmap.rs @@ -298,8 +298,12 @@ fn run_into( } if ctx.dry_run() { - print_dry_run(&dry_rows); - *summary = Summary::default(); + // A plain dry run prints its own table and discards the counts; an + // estimate keeps the per-type summary so the caller can render it. + if !ctx.estimate() { + print_dry_run(&dry_rows); + *summary = Summary::default(); + } return Ok(()); } @@ -399,6 +403,10 @@ fn reconcile_type( d.vanished.len() as u64, d.present.len() as u64, )); + // --estimate keeps the summary (counts only; JMAP would need a body + // fetch per object to size it, which an estimate does not do), so the + // new-object count has to land in the per-type row the caller pushes. + counts.fetched = d.new.len() as u64; return Ok(()); }