diff --git a/README.md b/README.md index 58b3a00..aa0bda4 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ takes `--dry-run` and then only reports the plan. An `import` also takes `--estimate`, which connects to the source and reports how much is new to fetch — counts, bytes and a rough time — before committing, and writes nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange -(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall -back to a plain dry-run. `inspect` reads an archive and changes nothing. +(`ews`) and a `takeout` (whose files it reads to count the mail, events and +contacts that are new); other sources fall back to a plain dry-run. `inspect` +reads an archive and changes nothing. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/src/cli.rs b/src/cli.rs index c7c91ab..dfe3fdc 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,9 +156,9 @@ struct GlobalArgs { help = "Preview an import: counts, sizes and a rough time, before any writes", long_help = "Preview an import before committing: for each type, how much is new since \ the archive was last filled, how many bytes that is, and a rough time.\n \ - Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \ - today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \ - never writes. On an export it is just --dry-run." + Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \ + events and contacts) today; other sources fall back to --dry-run with a note. It \ + implies --dry-run, so it never writes. On an export it is just --dry-run." )] estimate: bool, @@ -930,8 +930,8 @@ impl Action { // through the same command but report differently, so they stay // on the fallback for now. Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav), - // Takeout reads the mbox files to count mail; its calendars and - // contacts are not itemized yet. + // Takeout reads its .mbox, .ics and .vcf files to count the mail, + // events and contacts that are new, and sizes each. Action::ImportTakeout(c, _) => c.estimate, // EWS FindItem returns item:Size, so mail, calendars and contacts // are all counted and sized from the enumeration. diff --git a/src/sync/import_takeout/coordinator.rs b/src/sync/import_takeout/coordinator.rs index 9ba0d71..451f7c8 100644 --- a/src/sync/import_takeout/coordinator.rs +++ b/src/sync/import_takeout/coordinator.rs @@ -25,6 +25,7 @@ use super::labels::MappingOptions; use super::mail; use super::mbox::MessageIterator; use super::walk::{self, FileKind}; +use crate::sync::import_dav::calcard; #[derive(Debug)] pub struct TakeoutImportConfig { @@ -366,29 +367,54 @@ fn dry_run_summary( " existing in archive: emails={local_emails} events={local_events} cards={local_cards}" ); } - // --estimate: a plain dry run counts files; an estimate reads the mbox - // files to count the messages that are actually new and sum their bytes. - // Calendars and contacts are not itemized yet -- the file counts above - // stand in for them. - let email = if want_sizes { - estimate_mail(conn, source_id, walk_result, logger) + // --estimate: a plain dry run counts files; an estimate reads them to count + // the objects that are actually new since the archive was last filled and + // sum their bytes -- messages from the mboxes, events from the .ics files, + // cards from the .vcf files -- deduplicated by the same key the import uses. + let (email, events, cards) = if want_sizes { + ( + estimate_mail(conn, source_id, walk_result, logger), + estimate_calendar(conn, source_id, walk_result, logger), + estimate_contacts(conn, source_id, walk_result, logger), + ) } else { - TypeCounts::default() + Default::default() }; Summary { per_type: vec![ ("mailbox", TypeCounts::default()), ("email", email), ("calendar", TypeCounts::default()), - ("calendarevent", TypeCounts::default()), + ("calendarevent", events), ("addressbook", TypeCounts::default()), - ("contactcard", TypeCounts::default()), + ("contactcard", cards), ], retries_observed: 0, retry_after_sleeps: 0, } } +/// Loads the external ids (content hashes, event/card UIDs) the archive already +/// holds for `type_name`, as the dedup set an estimate scans against. An empty +/// set on a fresh or unreadable archive just means everything reads as new. +fn archived_keys( + conn: &Connection, + source_id: i64, + type_name: &str, + logger: Logger, +) -> HashSet { + if source_id < 0 { + return HashSet::new(); + } + match takeout_ids::all_for_type(conn, source_id, type_name) { + Ok(m) => m.into_keys().collect(), + Err(e) => { + logger.warn(&format!("dry-run: reading archived {type_name} ids: {e}")); + HashSet::new() + } + } +} + /// Scans the mbox files for `--estimate`: counts the messages not already in /// the archive and sums their bytes, deduplicating by the same content hash /// the import uses, so a message that appears in two mboxes counts once. It @@ -401,17 +427,7 @@ fn estimate_mail( logger: Logger, ) -> TypeCounts { let mut email = TypeCounts::default(); - let mut seen: HashSet = if source_id < 0 { - HashSet::new() - } else { - match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) { - Ok(m) => m.into_keys().collect(), - Err(e) => { - logger.warn(&format!("dry-run: reading archived message ids: {e}")); - HashSet::new() - } - } - }; + let mut seen = archived_keys(conn, source_id, takeout_ids::EMAIL, logger); for file in walk_result.by_kind(FileKind::Mbox) { let reader = match File::open(&file.path) { Ok(f) => BufReader::new(f), @@ -448,6 +464,107 @@ fn estimate_mail( email } +/// Scans the `.ics` files for `--estimate`: counts the events not already in +/// the archive and sums the bytes of the JSCalendar record each would store. +/// It derives each event's UID exactly as the import does -- the event's own +/// UID, or one synthesised from the file path and ordinal when it has none -- +/// so the same event in two files, or one already imported, counts once. It +/// parses the iCalendar but writes nothing. +fn estimate_calendar( + conn: &Connection, + source_id: i64, + walk_result: &walk::WalkResult, + logger: Logger, +) -> TypeCounts { + let mut events = TypeCounts::default(); + let mut seen = archived_keys(conn, source_id, takeout_ids::CALENDAR_EVENT, logger); + for file in walk_result.by_kind(FileKind::Ics) { + let text = match std::fs::read_to_string(&file.path) { + Ok(s) => s, + Err(e) => { + logger.warn(&format!("dry-run read {:?}: {e}", file.path)); + events.failed += 1; + continue; + } + }; + let entries = match calcard::ical_to_jscalendar_entries(&text) { + Ok(v) => v, + Err(calcard::CalcardError::NoEntries) => continue, + Err(e) => { + logger.warn(&format!("dry-run {:?}: iCalendar parse: {e}", file.path)); + events.failed += 1; + continue; + } + }; + for (ordinal, mut entry) in entries.into_iter().enumerate() { + let (_is_draft, _use_default_alerts, uid_field) = + calcard::strip_extracted_fields_from_event(&mut entry.data); + let uid = match uid_field + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + Some(u) => u.to_owned(), + None => { + calcard::synthesise_uid(&format!("{}#{ordinal}", file.path.to_string_lossy())) + } + }; + let size = entry.data.to_string().len() as u64; + if seen.insert(uid) { + events.created += 1; + events.fetched += 1; + events.bytes += size; + } else { + events.skipped += 1; + } + } + } + events +} + +/// Scans the `.vcf` files for `--estimate`: counts the contacts not already in +/// the archive and sums the bytes of the JSContact record each would store, +/// deduplicating by the card's UID, the same key the import keeps. It parses +/// the vCards but writes nothing. +fn estimate_contacts( + conn: &Connection, + source_id: i64, + walk_result: &walk::WalkResult, + logger: Logger, +) -> TypeCounts { + let mut cards = TypeCounts::default(); + let mut seen = archived_keys(conn, source_id, takeout_ids::CONTACT_CARD, logger); + for file in walk_result.by_kind(FileKind::Vcf) { + let text = match std::fs::read_to_string(&file.path) { + Ok(s) => s, + Err(e) => { + logger.warn(&format!("dry-run read {:?}: {e}", file.path)); + cards.failed += 1; + continue; + } + }; + for chunk in contacts::split_vcards(&text) { + match calcard::vcard_to_jscontact(&chunk, &chunk) { + Ok(card) => { + let size = card.data.to_string().len() as u64; + if seen.insert(card.uid) { + cards.created += 1; + cards.fetched += 1; + cards.bytes += size; + } else { + cards.skipped += 1; + } + } + Err(e) => { + logger.warn(&format!("dry-run {:?}: vCard parse: {e}", file.path)); + cards.failed += 1; + } + } + } + } + cards +} + fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 { conn.query_row( "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2", diff --git a/tests/sync_takeout.rs b/tests/sync_takeout.rs index b00e71c..308b164 100644 --- a/tests/sync_takeout.rs +++ b/tests/sync_takeout.rs @@ -282,7 +282,7 @@ fn synthetic_takeout_imports_expected_row_counts() { } #[test] -fn estimate_counts_new_mail_and_sizes_it() { +fn estimate_counts_new_objects_and_sizes_them() { let td = TempDir::new().unwrap(); SyntheticTakeout::build(td.path()); let archive = td.path().join("a.sqlite"); @@ -293,27 +293,58 @@ fn estimate_counts_new_mail_and_sizes_it() { c.estimate = true; c }; - let email_of = |summary: &inbuxa_migrate::sync::Summary| { + let by = |summary: &inbuxa_migrate::sync::Summary, t: &str| { summary .per_type .iter() - .find(|(n, _)| *n == "email") + .find(|(n, _)| *n == t) .unwrap() .1 .clone() }; - // A fresh archive: every synthetic message is new, and it is sized. + // A fresh archive: every synthetic object is new, across all three kinds, + // and each kind is sized. let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap(); - let email = email_of(&first); + + let email = by(&first, "email"); assert_eq!(email.fetched, 9, "all 9 synthetic messages are new"); assert!(email.bytes > 0, "new mail is sized: {}", email.bytes); assert_eq!(email.failed, 0); - // After a real import, a re-estimate finds nothing new. + let events = by(&first, "calendarevent"); + assert!(events.fetched >= 2, "the .ics events are new: {events:?}"); + assert!(events.bytes > 0, "new events are sized: {}", events.bytes); + assert_eq!(events.failed, 0); + + // The two .vcf files carry the same cards; UID dedup collapses them to the + // four the import creates, so the estimate must report four, not eight. + let cards = by(&first, "contactcard"); + assert_eq!( + cards.fetched, 4, + "cards dedup by UID across the two .vcf files" + ); + assert!(cards.bytes > 0, "new cards are sized: {}", cards.bytes); + assert_eq!(cards.failed, 0); + + // After a real import, a re-estimate finds nothing new of any kind. run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap(); let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap(); - assert_eq!(email_of(&again).fetched, 0, "nothing new after importing"); + assert_eq!( + by(&again, "email").fetched, + 0, + "no new mail after importing" + ); + assert_eq!( + by(&again, "calendarevent").fetched, + 0, + "no new events after importing" + ); + assert_eq!( + by(&again, "contactcard").fetched, + 0, + "no new cards after importing" + ); } #[test]