From 6559b9812de1dfe409a32edf2cbce571198d9ae6 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 12 Nov 2026 10:00:58 -0700 Subject: [PATCH] estimate: itemize a Takeout's calendars and contacts too Takeout's --estimate counted and sized the mail but left calendars and contacts to the plain file counts. Now it reads the .ics and .vcf files as well: it counts the events and contacts that are new since the archive was last filled and sums the bytes of the record each would store, writing nothing. Each kind dedupes by the same key the import keeps -- events by their UID (synthesised from the file path and ordinal when an event has none, exactly as the import does), contacts by the card UID -- so an object that appears in two files, or one already imported, counts once. A re-estimate of a part-filled archive reports only what is left. The archived-id lookup the three scans share is pulled into one helper, archived_keys. The synthetic-Takeout estimate test now asserts new events and contacts are counted and sized, and that a re-estimate after import finds nothing new of any kind. --- README.md | 5 +- src/cli.rs | 10 +- src/sync/import_takeout/coordinator.rs | 157 +++++++++++++++++++++---- tests/sync_takeout.rs | 45 +++++-- 4 files changed, 183 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 58b3a00..aa0bda4 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ takes `--dry-run` and then only reports the plan. An `import` also takes `--estimate`, which connects to the source and reports how much is new to fetch — counts, bytes and a rough time — before committing, and writes nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange -(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall -back to a plain dry-run. `inspect` reads an archive and changes nothing. +(`ews`) and a `takeout` (whose files it reads to count the mail, events and +contacts that are new); other sources fall back to a plain dry-run. `inspect` +reads an archive and changes nothing. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/src/cli.rs b/src/cli.rs index c7c91ab..dfe3fdc 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,9 +156,9 @@ struct GlobalArgs { help = "Preview an import: counts, sizes and a rough time, before any writes", long_help = "Preview an import before committing: for each type, how much is new since \ the archive was last filled, how many bytes that is, and a rough time.\n \ - Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \ - today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \ - never writes. On an export it is just --dry-run." + Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \ + events and contacts) today; other sources fall back to --dry-run with a note. It \ + implies --dry-run, so it never writes. On an export it is just --dry-run." )] estimate: bool, @@ -930,8 +930,8 @@ impl Action { // through the same command but report differently, so they stay // on the fallback for now. Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav), - // Takeout reads the mbox files to count mail; its calendars and - // contacts are not itemized yet. + // Takeout reads its .mbox, .ics and .vcf files to count the mail, + // events and contacts that are new, and sizes each. Action::ImportTakeout(c, _) => c.estimate, // EWS FindItem returns item:Size, so mail, calendars and contacts // are all counted and sized from the enumeration. diff --git a/src/sync/import_takeout/coordinator.rs b/src/sync/import_takeout/coordinator.rs index 9ba0d71..451f7c8 100644 --- a/src/sync/import_takeout/coordinator.rs +++ b/src/sync/import_takeout/coordinator.rs @@ -25,6 +25,7 @@ use super::labels::MappingOptions; use super::mail; use super::mbox::MessageIterator; use super::walk::{self, FileKind}; +use crate::sync::import_dav::calcard; #[derive(Debug)] pub struct TakeoutImportConfig { @@ -366,29 +367,54 @@ fn dry_run_summary( " existing in archive: emails={local_emails} events={local_events} cards={local_cards}" ); } - // --estimate: a plain dry run counts files; an estimate reads the mbox - // files to count the messages that are actually new and sum their bytes. - // Calendars and contacts are not itemized yet -- the file counts above - // stand in for them. - let email = if want_sizes { - estimate_mail(conn, source_id, walk_result, logger) + // --estimate: a plain dry run counts files; an estimate reads them to count + // the objects that are actually new since the archive was last filled and + // sum their bytes -- messages from the mboxes, events from the .ics files, + // cards from the .vcf files -- deduplicated by the same key the import uses. + let (email, events, cards) = if want_sizes { + ( + estimate_mail(conn, source_id, walk_result, logger), + estimate_calendar(conn, source_id, walk_result, logger), + estimate_contacts(conn, source_id, walk_result, logger), + ) } else { - TypeCounts::default() + Default::default() }; Summary { per_type: vec![ ("mailbox", TypeCounts::default()), ("email", email), ("calendar", TypeCounts::default()), - ("calendarevent", TypeCounts::default()), + ("calendarevent", events), ("addressbook", TypeCounts::default()), - ("contactcard", TypeCounts::default()), + ("contactcard", cards), ], retries_observed: 0, retry_after_sleeps: 0, } } +/// Loads the external ids (content hashes, event/card UIDs) the archive already +/// holds for `type_name`, as the dedup set an estimate scans against. An empty +/// set on a fresh or unreadable archive just means everything reads as new. +fn archived_keys( + conn: &Connection, + source_id: i64, + type_name: &str, + logger: Logger, +) -> HashSet { + if source_id < 0 { + return HashSet::new(); + } + match takeout_ids::all_for_type(conn, source_id, type_name) { + Ok(m) => m.into_keys().collect(), + Err(e) => { + logger.warn(&format!("dry-run: reading archived {type_name} ids: {e}")); + HashSet::new() + } + } +} + /// Scans the mbox files for `--estimate`: counts the messages not already in /// the archive and sums their bytes, deduplicating by the same content hash /// the import uses, so a message that appears in two mboxes counts once. It @@ -401,17 +427,7 @@ fn estimate_mail( logger: Logger, ) -> TypeCounts { let mut email = TypeCounts::default(); - let mut seen: HashSet = if source_id < 0 { - HashSet::new() - } else { - match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) { - Ok(m) => m.into_keys().collect(), - Err(e) => { - logger.warn(&format!("dry-run: reading archived message ids: {e}")); - HashSet::new() - } - } - }; + let mut seen = archived_keys(conn, source_id, takeout_ids::EMAIL, logger); for file in walk_result.by_kind(FileKind::Mbox) { let reader = match File::open(&file.path) { Ok(f) => BufReader::new(f), @@ -448,6 +464,107 @@ fn estimate_mail( email } +/// Scans the `.ics` files for `--estimate`: counts the events not already in +/// the archive and sums the bytes of the JSCalendar record each would store. +/// It derives each event's UID exactly as the import does -- the event's own +/// UID, or one synthesised from the file path and ordinal when it has none -- +/// so the same event in two files, or one already imported, counts once. It +/// parses the iCalendar but writes nothing. +fn estimate_calendar( + conn: &Connection, + source_id: i64, + walk_result: &walk::WalkResult, + logger: Logger, +) -> TypeCounts { + let mut events = TypeCounts::default(); + let mut seen = archived_keys(conn, source_id, takeout_ids::CALENDAR_EVENT, logger); + for file in walk_result.by_kind(FileKind::Ics) { + let text = match std::fs::read_to_string(&file.path) { + Ok(s) => s, + Err(e) => { + logger.warn(&format!("dry-run read {:?}: {e}", file.path)); + events.failed += 1; + continue; + } + }; + let entries = match calcard::ical_to_jscalendar_entries(&text) { + Ok(v) => v, + Err(calcard::CalcardError::NoEntries) => continue, + Err(e) => { + logger.warn(&format!("dry-run {:?}: iCalendar parse: {e}", file.path)); + events.failed += 1; + continue; + } + }; + for (ordinal, mut entry) in entries.into_iter().enumerate() { + let (_is_draft, _use_default_alerts, uid_field) = + calcard::strip_extracted_fields_from_event(&mut entry.data); + let uid = match uid_field + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + Some(u) => u.to_owned(), + None => { + calcard::synthesise_uid(&format!("{}#{ordinal}", file.path.to_string_lossy())) + } + }; + let size = entry.data.to_string().len() as u64; + if seen.insert(uid) { + events.created += 1; + events.fetched += 1; + events.bytes += size; + } else { + events.skipped += 1; + } + } + } + events +} + +/// Scans the `.vcf` files for `--estimate`: counts the contacts not already in +/// the archive and sums the bytes of the JSContact record each would store, +/// deduplicating by the card's UID, the same key the import keeps. It parses +/// the vCards but writes nothing. +fn estimate_contacts( + conn: &Connection, + source_id: i64, + walk_result: &walk::WalkResult, + logger: Logger, +) -> TypeCounts { + let mut cards = TypeCounts::default(); + let mut seen = archived_keys(conn, source_id, takeout_ids::CONTACT_CARD, logger); + for file in walk_result.by_kind(FileKind::Vcf) { + let text = match std::fs::read_to_string(&file.path) { + Ok(s) => s, + Err(e) => { + logger.warn(&format!("dry-run read {:?}: {e}", file.path)); + cards.failed += 1; + continue; + } + }; + for chunk in contacts::split_vcards(&text) { + match calcard::vcard_to_jscontact(&chunk, &chunk) { + Ok(card) => { + let size = card.data.to_string().len() as u64; + if seen.insert(card.uid) { + cards.created += 1; + cards.fetched += 1; + cards.bytes += size; + } else { + cards.skipped += 1; + } + } + Err(e) => { + logger.warn(&format!("dry-run {:?}: vCard parse: {e}", file.path)); + cards.failed += 1; + } + } + } + } + cards +} + fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 { conn.query_row( "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2", diff --git a/tests/sync_takeout.rs b/tests/sync_takeout.rs index b00e71c..308b164 100644 --- a/tests/sync_takeout.rs +++ b/tests/sync_takeout.rs @@ -282,7 +282,7 @@ fn synthetic_takeout_imports_expected_row_counts() { } #[test] -fn estimate_counts_new_mail_and_sizes_it() { +fn estimate_counts_new_objects_and_sizes_them() { let td = TempDir::new().unwrap(); SyntheticTakeout::build(td.path()); let archive = td.path().join("a.sqlite"); @@ -293,27 +293,58 @@ fn estimate_counts_new_mail_and_sizes_it() { c.estimate = true; c }; - let email_of = |summary: &inbuxa_migrate::sync::Summary| { + let by = |summary: &inbuxa_migrate::sync::Summary, t: &str| { summary .per_type .iter() - .find(|(n, _)| *n == "email") + .find(|(n, _)| *n == t) .unwrap() .1 .clone() }; - // A fresh archive: every synthetic message is new, and it is sized. + // A fresh archive: every synthetic object is new, across all three kinds, + // and each kind is sized. let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap(); - let email = email_of(&first); + + let email = by(&first, "email"); assert_eq!(email.fetched, 9, "all 9 synthetic messages are new"); assert!(email.bytes > 0, "new mail is sized: {}", email.bytes); assert_eq!(email.failed, 0); - // After a real import, a re-estimate finds nothing new. + let events = by(&first, "calendarevent"); + assert!(events.fetched >= 2, "the .ics events are new: {events:?}"); + assert!(events.bytes > 0, "new events are sized: {}", events.bytes); + assert_eq!(events.failed, 0); + + // The two .vcf files carry the same cards; UID dedup collapses them to the + // four the import creates, so the estimate must report four, not eight. + let cards = by(&first, "contactcard"); + assert_eq!( + cards.fetched, 4, + "cards dedup by UID across the two .vcf files" + ); + assert!(cards.bytes > 0, "new cards are sized: {}", cards.bytes); + assert_eq!(cards.failed, 0); + + // After a real import, a re-estimate finds nothing new of any kind. run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap(); let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap(); - assert_eq!(email_of(&again).fetched, 0, "nothing new after importing"); + assert_eq!( + by(&again, "email").fetched, + 0, + "no new mail after importing" + ); + assert_eq!( + by(&again, "calendarevent").fetched, + 0, + "no new events after importing" + ); + assert_eq!( + by(&again, "contactcard").fetched, + 0, + "no new cards after importing" + ); } #[test] -- 2.54.0