diff --git a/README.md b/README.md index 492425a..db5fa50 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ later `import` into the same archive adds what arrived since. Every command takes `--dry-run` and then only reports the plan. An `import` also takes `--estimate`, which connects to the source and reports how much is new to fetch — counts, bytes and a rough time — before committing, and writes -nothing. It reports sizes for `imap`, `maildir`, `caldav` and `carddav`; other -sources fall back to a plain dry-run. `inspect` reads an archive and changes -nothing. +nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav` and a +`takeout`'s mail (which it reads to count); other sources fall back to a plain +dry-run. `inspect` reads an archive and changes nothing. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/src/cli.rs b/src/cli.rs index 1a4f4e9..7c5ac8f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,9 +156,9 @@ struct GlobalArgs { help = "Preview an import: counts, sizes and a rough time, before any writes", long_help = "Preview an import before committing: for each type, how much is new since \ the archive was last filled, how many bytes that is, and a rough time.\n \ - Available for imap, maildir, caldav and carddav today; other sources fall back to \ - --dry-run with a note. It implies --dry-run, so it never writes. On an export it is \ - just --dry-run." + Available for imap, maildir, caldav, carddav and takeout (its mail) today; other \ + sources fall back to --dry-run with a note. It implies --dry-run, so it never writes. \ + On an export it is just --dry-run." )] estimate: bool, @@ -930,6 +930,9 @@ impl Action { // through the same command but report differently, so they stay // on the fallback for now. Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav), + // Takeout reads the mbox files to count mail; its calendars and + // contacts are not itemized yet. + Action::ImportTakeout(c, _) => c.estimate, _ => false, } } diff --git a/src/main.rs b/src/main.rs index bffdb9f..bc2c2ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ fn run() -> i32 { if action.estimate_requested() && !do_estimate { eprintln!( "note: --estimate isn't available for this source yet; running --dry-run instead \ - (its plan is below). Estimates work for imap, maildir, caldav and carddav today." + (its plan is below). Estimates work for imap, maildir, caldav, carddav and takeout today." ); } let mut quiet_report = false; diff --git a/src/sync/import_takeout/coordinator.rs b/src/sync/import_takeout/coordinator.rs index f6f8743..9ba0d71 100644 --- a/src/sync/import_takeout/coordinator.rs +++ b/src/sync/import_takeout/coordinator.rs @@ -4,7 +4,9 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::BufReader; use std::path::{Path, PathBuf}; use rusqlite::{Connection, params}; @@ -21,6 +23,7 @@ use super::calendar; use super::contacts; use super::labels::MappingOptions; use super::mail; +use super::mbox::MessageIterator; use super::walk::{self, FileKind}; #[derive(Debug)] @@ -104,7 +107,7 @@ fn run_into( }; if common.dry_run { - *summary = dry_run_summary(&conn, source_id, &walk_result, logger); + *summary = dry_run_summary(&conn, source_id, &walk_result, common.estimate, logger); return Ok(()); } @@ -336,6 +339,7 @@ fn dry_run_summary( conn: &Connection, source_id: i64, walk_result: &walk::WalkResult, + want_sizes: bool, logger: Logger, ) -> Summary { let mbox_n = walk_result.by_kind(FileKind::Mbox).count(); @@ -362,10 +366,19 @@ fn dry_run_summary( " existing in archive: emails={local_emails} events={local_events} cards={local_cards}" ); } + // --estimate: a plain dry run counts files; an estimate reads the mbox + // files to count the messages that are actually new and sum their bytes. + // Calendars and contacts are not itemized yet -- the file counts above + // stand in for them. + let email = if want_sizes { + estimate_mail(conn, source_id, walk_result, logger) + } else { + TypeCounts::default() + }; Summary { per_type: vec![ ("mailbox", TypeCounts::default()), - ("email", TypeCounts::default()), + ("email", email), ("calendar", TypeCounts::default()), ("calendarevent", TypeCounts::default()), ("addressbook", TypeCounts::default()), @@ -376,6 +389,65 @@ fn dry_run_summary( } } +/// Scans the mbox files for `--estimate`: counts the messages not already in +/// the archive and sums their bytes, deduplicating by the same content hash +/// the import uses, so a message that appears in two mboxes counts once. It +/// reads the mbox bodies -- there is no metadata to ask, the files are the +/// source -- but parses no MIME and writes nothing. +fn estimate_mail( + conn: &Connection, + source_id: i64, + walk_result: &walk::WalkResult, + logger: Logger, +) -> TypeCounts { + let mut email = TypeCounts::default(); + let mut seen: HashSet = if source_id < 0 { + HashSet::new() + } else { + match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) { + Ok(m) => m.into_keys().collect(), + Err(e) => { + logger.warn(&format!("dry-run: reading archived message ids: {e}")); + HashSet::new() + } + } + }; + for file in walk_result.by_kind(FileKind::Mbox) { + let reader = match File::open(&file.path) { + Ok(f) => BufReader::new(f), + Err(e) => { + logger.warn(&format!("dry-run open {:?}: {e}", file.path)); + email.failed += 1; + continue; + } + }; + let mut iter = MessageIterator::new(reader); + loop { + match iter.next() { + None => break, + Some(Err(e)) => { + logger.warn(&format!("dry-run {:?}: mbox io: {e}", file.path)); + email.failed += 1; + break; + } + Some(Ok(msg)) => { + let hash = blake3::hash(msg.contents()).to_hex().to_string(); + // insert() is true only the first time a hash is seen, so + // archived and duplicate messages are not counted. + if seen.insert(hash) { + email.created += 1; + email.fetched += 1; + email.bytes += msg.contents().len() as u64; + } else { + email.skipped += 1; + } + } + } + } + } + email +} + fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 { conn.query_row( "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2", diff --git a/tests/sync_takeout.rs b/tests/sync_takeout.rs index d17588e..b00e71c 100644 --- a/tests/sync_takeout.rs +++ b/tests/sync_takeout.rs @@ -281,6 +281,41 @@ fn synthetic_takeout_imports_expected_row_counts() { assert_eq!(by("contactcard").failed, 0); } +#[test] +fn estimate_counts_new_mail_and_sizes_it() { + let td = TempDir::new().unwrap(); + SyntheticTakeout::build(td.path()); + let archive = td.path().join("a.sqlite"); + + let estimate_cfg = |archive: &Path| { + let mut c = common_for(archive); + c.dry_run = true; + c.estimate = true; + c + }; + let email_of = |summary: &inbuxa_migrate::sync::Summary| { + summary + .per_type + .iter() + .find(|(n, _)| *n == "email") + .unwrap() + .1 + .clone() + }; + + // A fresh archive: every synthetic message is new, and it is sized. + let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap(); + let email = email_of(&first); + assert_eq!(email.fetched, 9, "all 9 synthetic messages are new"); + assert!(email.bytes > 0, "new mail is sized: {}", email.bytes); + assert_eq!(email.failed, 0); + + // After a real import, a re-estimate finds nothing new. + run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap(); + let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap(); + assert_eq!(email_of(&again).fetched, 0, "nothing new after importing"); +} + #[test] fn label_mapping_produces_expected_mailbox_tree() { let td = TempDir::new().unwrap();