estimate: count and size a Takeout's mail #69

Merged
jcoffey-dev merged 1 commits from feat/estimate-takeout into main 2026-11-12 16:42:41 +00:00
5 changed files with 120 additions and 10 deletions

No files matched your search

+3 -3
View File
@@ -38,9 +38,9 @@ later `import` into the same archive adds what arrived since. Every command
takes `--dry-run` and then only reports the plan. An `import` also takes
`--estimate`, which connects to the source and reports how much is new to
fetch — counts, bytes and a rough time — before committing, and writes
nothing. It reports sizes for `imap`, `maildir`, `caldav` and `carddav`; other
sources fall back to a plain dry-run. `inspect` reads an archive and changes
nothing.
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav` and a
`takeout`'s mail (which it reads to count); other sources fall back to a plain
dry-run. `inspect` reads an archive and changes nothing.
The archive is also a backup. Import on a schedule, keep the file, and
restore it later with `export`. An archive remembers which account filled it
+6 -3
View File
@@ -156,9 +156,9 @@ struct GlobalArgs {
help = "Preview an import: counts, sizes and a rough time, before any writes",
long_help = "Preview an import before committing: for each type, how much is new since \
the archive was last filled, how many bytes that is, and a rough time.\n \
Available for imap, maildir, caldav and carddav today; other sources fall back to \
--dry-run with a note. It implies --dry-run, so it never writes. On an export it is \
just --dry-run."
Available for imap, maildir, caldav, carddav and takeout (its mail) today; other \
sources fall back to --dry-run with a note. It implies --dry-run, so it never writes. \
On an export it is just --dry-run."
)]
estimate: bool,
@@ -930,6 +930,9 @@ impl Action {
// through the same command but report differently, so they stay
// on the fallback for now.
Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav),
// Takeout reads the mbox files to count mail; its calendars and
// contacts are not itemized yet.
Action::ImportTakeout(c, _) => c.estimate,
_ => false,
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ fn run() -> i32 {
if action.estimate_requested() && !do_estimate {
eprintln!(
"note: --estimate isn't available for this source yet; running --dry-run instead \
(its plan is below). Estimates work for imap, maildir, caldav and carddav today."
(its plan is below). Estimates work for imap, maildir, caldav, carddav and takeout today."
);
}
let mut quiet_report = false;
+75 -3
View File
@@ -4,7 +4,9 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use rusqlite::{Connection, params};
@@ -21,6 +23,7 @@ use super::calendar;
use super::contacts;
use super::labels::MappingOptions;
use super::mail;
use super::mbox::MessageIterator;
use super::walk::{self, FileKind};
#[derive(Debug)]
@@ -104,7 +107,7 @@ fn run_into(
};
if common.dry_run {
*summary = dry_run_summary(&conn, source_id, &walk_result, logger);
*summary = dry_run_summary(&conn, source_id, &walk_result, common.estimate, logger);
return Ok(());
}
@@ -336,6 +339,7 @@ fn dry_run_summary(
conn: &Connection,
source_id: i64,
walk_result: &walk::WalkResult,
want_sizes: bool,
logger: Logger,
) -> Summary {
let mbox_n = walk_result.by_kind(FileKind::Mbox).count();
@@ -362,10 +366,19 @@ fn dry_run_summary(
" existing in archive: emails={local_emails} events={local_events} cards={local_cards}"
);
}
// --estimate: a plain dry run counts files; an estimate reads the mbox
// files to count the messages that are actually new and sum their bytes.
// Calendars and contacts are not itemized yet -- the file counts above
// stand in for them.
let email = if want_sizes {
estimate_mail(conn, source_id, walk_result, logger)
} else {
TypeCounts::default()
};
Summary {
per_type: vec![
("mailbox", TypeCounts::default()),
("email", TypeCounts::default()),
("email", email),
("calendar", TypeCounts::default()),
("calendarevent", TypeCounts::default()),
("addressbook", TypeCounts::default()),
@@ -376,6 +389,65 @@ fn dry_run_summary(
}
}
/// Scans the mbox files for `--estimate`: counts the messages not already in
/// the archive and sums their bytes, deduplicating by the same content hash
/// the import uses, so a message that appears in two mboxes counts once. It
/// reads the mbox bodies -- there is no metadata to ask, the files are the
/// source -- but parses no MIME and writes nothing.
fn estimate_mail(
conn: &Connection,
source_id: i64,
walk_result: &walk::WalkResult,
logger: Logger,
) -> TypeCounts {
let mut email = TypeCounts::default();
let mut seen: HashSet<String> = if source_id < 0 {
HashSet::new()
} else {
match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) {
Ok(m) => m.into_keys().collect(),
Err(e) => {
logger.warn(&format!("dry-run: reading archived message ids: {e}"));
HashSet::new()
}
}
};
for file in walk_result.by_kind(FileKind::Mbox) {
let reader = match File::open(&file.path) {
Ok(f) => BufReader::new(f),
Err(e) => {
logger.warn(&format!("dry-run open {:?}: {e}", file.path));
email.failed += 1;
continue;
}
};
let mut iter = MessageIterator::new(reader);
loop {
match iter.next() {
None => break,
Some(Err(e)) => {
logger.warn(&format!("dry-run {:?}: mbox io: {e}", file.path));
email.failed += 1;
break;
}
Some(Ok(msg)) => {
let hash = blake3::hash(msg.contents()).to_hex().to_string();
// insert() is true only the first time a hash is seen, so
// archived and duplicate messages are not counted.
if seen.insert(hash) {
email.created += 1;
email.fetched += 1;
email.bytes += msg.contents().len() as u64;
} else {
email.skipped += 1;
}
}
}
}
}
email
}
fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 {
conn.query_row(
"SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2",
+35
View File
@@ -281,6 +281,41 @@ fn synthetic_takeout_imports_expected_row_counts() {
assert_eq!(by("contactcard").failed, 0);
}
#[test]
fn estimate_counts_new_mail_and_sizes_it() {
let td = TempDir::new().unwrap();
SyntheticTakeout::build(td.path());
let archive = td.path().join("a.sqlite");
let estimate_cfg = |archive: &Path| {
let mut c = common_for(archive);
c.dry_run = true;
c.estimate = true;
c
};
let email_of = |summary: &inbuxa_migrate::sync::Summary| {
summary
.per_type
.iter()
.find(|(n, _)| *n == "email")
.unwrap()
.1
.clone()
};
// A fresh archive: every synthetic message is new, and it is sized.
let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
let email = email_of(&first);
assert_eq!(email.fetched, 9, "all 9 synthetic messages are new");
assert!(email.bytes > 0, "new mail is sized: {}", email.bytes);
assert_eq!(email.failed, 0);
// After a real import, a re-estimate finds nothing new.
run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap();
let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
assert_eq!(email_of(&again).fetched, 0, "nothing new after importing");
}
#[test]
fn label_mapping_produces_expected_mailbox_tree() {
let td = TempDir::new().unwrap();