estimate: count and size a Takeout's mail #69
No files matched your search
@@ -38,9 +38,9 @@ later `import` into the same archive adds what arrived since. Every command
|
||||
takes `--dry-run` and then only reports the plan. An `import` also takes
|
||||
`--estimate`, which connects to the source and reports how much is new to
|
||||
fetch — counts, bytes and a rough time — before committing, and writes
|
||||
nothing. It reports sizes for `imap`, `maildir`, `caldav` and `carddav`; other
|
||||
sources fall back to a plain dry-run. `inspect` reads an archive and changes
|
||||
nothing.
|
||||
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav` and a
|
||||
`takeout`'s mail (which it reads to count); other sources fall back to a plain
|
||||
dry-run. `inspect` reads an archive and changes nothing.
|
||||
|
||||
The archive is also a backup. Import on a schedule, keep the file, and
|
||||
restore it later with `export`. An archive remembers which account filled it
|
||||
|
||||
+6
-3
@@ -156,9 +156,9 @@ struct GlobalArgs {
|
||||
help = "Preview an import: counts, sizes and a rough time, before any writes",
|
||||
long_help = "Preview an import before committing: for each type, how much is new since \
|
||||
the archive was last filled, how many bytes that is, and a rough time.\n \
|
||||
Available for imap, maildir, caldav and carddav today; other sources fall back to \
|
||||
--dry-run with a note. It implies --dry-run, so it never writes. On an export it is \
|
||||
just --dry-run."
|
||||
Available for imap, maildir, caldav, carddav and takeout (its mail) today; other \
|
||||
sources fall back to --dry-run with a note. It implies --dry-run, so it never writes. \
|
||||
On an export it is just --dry-run."
|
||||
)]
|
||||
estimate: bool,
|
||||
|
||||
@@ -930,6 +930,9 @@ impl Action {
|
||||
// through the same command but report differently, so they stay
|
||||
// on the fallback for now.
|
||||
Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav),
|
||||
// Takeout reads the mbox files to count mail; its calendars and
|
||||
// contacts are not itemized yet.
|
||||
Action::ImportTakeout(c, _) => c.estimate,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ fn run() -> i32 {
|
||||
if action.estimate_requested() && !do_estimate {
|
||||
eprintln!(
|
||||
"note: --estimate isn't available for this source yet; running --dry-run instead \
|
||||
(its plan is below). Estimates work for imap, maildir, caldav and carddav today."
|
||||
(its plan is below). Estimates work for imap, maildir, caldav, carddav and takeout today."
|
||||
);
|
||||
}
|
||||
let mut quiet_report = false;
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
@@ -21,6 +23,7 @@ use super::calendar;
|
||||
use super::contacts;
|
||||
use super::labels::MappingOptions;
|
||||
use super::mail;
|
||||
use super::mbox::MessageIterator;
|
||||
use super::walk::{self, FileKind};
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -104,7 +107,7 @@ fn run_into(
|
||||
};
|
||||
|
||||
if common.dry_run {
|
||||
*summary = dry_run_summary(&conn, source_id, &walk_result, logger);
|
||||
*summary = dry_run_summary(&conn, source_id, &walk_result, common.estimate, logger);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -336,6 +339,7 @@ fn dry_run_summary(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
walk_result: &walk::WalkResult,
|
||||
want_sizes: bool,
|
||||
logger: Logger,
|
||||
) -> Summary {
|
||||
let mbox_n = walk_result.by_kind(FileKind::Mbox).count();
|
||||
@@ -362,10 +366,19 @@ fn dry_run_summary(
|
||||
" existing in archive: emails={local_emails} events={local_events} cards={local_cards}"
|
||||
);
|
||||
}
|
||||
// --estimate: a plain dry run counts files; an estimate reads the mbox
|
||||
// files to count the messages that are actually new and sum their bytes.
|
||||
// Calendars and contacts are not itemized yet -- the file counts above
|
||||
// stand in for them.
|
||||
let email = if want_sizes {
|
||||
estimate_mail(conn, source_id, walk_result, logger)
|
||||
} else {
|
||||
TypeCounts::default()
|
||||
};
|
||||
Summary {
|
||||
per_type: vec![
|
||||
("mailbox", TypeCounts::default()),
|
||||
("email", TypeCounts::default()),
|
||||
("email", email),
|
||||
("calendar", TypeCounts::default()),
|
||||
("calendarevent", TypeCounts::default()),
|
||||
("addressbook", TypeCounts::default()),
|
||||
@@ -376,6 +389,65 @@ fn dry_run_summary(
|
||||
}
|
||||
}
|
||||
|
||||
/// Scans the mbox files for `--estimate`: counts the messages not already in
|
||||
/// the archive and sums their bytes, deduplicating by the same content hash
|
||||
/// the import uses, so a message that appears in two mboxes counts once. It
|
||||
/// reads the mbox bodies -- there is no metadata to ask, the files are the
|
||||
/// source -- but parses no MIME and writes nothing.
|
||||
fn estimate_mail(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
walk_result: &walk::WalkResult,
|
||||
logger: Logger,
|
||||
) -> TypeCounts {
|
||||
let mut email = TypeCounts::default();
|
||||
let mut seen: HashSet<String> = if source_id < 0 {
|
||||
HashSet::new()
|
||||
} else {
|
||||
match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) {
|
||||
Ok(m) => m.into_keys().collect(),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run: reading archived message ids: {e}"));
|
||||
HashSet::new()
|
||||
}
|
||||
}
|
||||
};
|
||||
for file in walk_result.by_kind(FileKind::Mbox) {
|
||||
let reader = match File::open(&file.path) {
|
||||
Ok(f) => BufReader::new(f),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run open {:?}: {e}", file.path));
|
||||
email.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut iter = MessageIterator::new(reader);
|
||||
loop {
|
||||
match iter.next() {
|
||||
None => break,
|
||||
Some(Err(e)) => {
|
||||
logger.warn(&format!("dry-run {:?}: mbox io: {e}", file.path));
|
||||
email.failed += 1;
|
||||
break;
|
||||
}
|
||||
Some(Ok(msg)) => {
|
||||
let hash = blake3::hash(msg.contents()).to_hex().to_string();
|
||||
// insert() is true only the first time a hash is seen, so
|
||||
// archived and duplicate messages are not counted.
|
||||
if seen.insert(hash) {
|
||||
email.created += 1;
|
||||
email.fetched += 1;
|
||||
email.bytes += msg.contents().len() as u64;
|
||||
} else {
|
||||
email.skipped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
email
|
||||
}
|
||||
|
||||
fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 {
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2",
|
||||
|
||||
@@ -281,6 +281,41 @@ fn synthetic_takeout_imports_expected_row_counts() {
|
||||
assert_eq!(by("contactcard").failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_counts_new_mail_and_sizes_it() {
|
||||
let td = TempDir::new().unwrap();
|
||||
SyntheticTakeout::build(td.path());
|
||||
let archive = td.path().join("a.sqlite");
|
||||
|
||||
let estimate_cfg = |archive: &Path| {
|
||||
let mut c = common_for(archive);
|
||||
c.dry_run = true;
|
||||
c.estimate = true;
|
||||
c
|
||||
};
|
||||
let email_of = |summary: &inbuxa_migrate::sync::Summary| {
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(n, _)| *n == "email")
|
||||
.unwrap()
|
||||
.1
|
||||
.clone()
|
||||
};
|
||||
|
||||
// A fresh archive: every synthetic message is new, and it is sized.
|
||||
let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
|
||||
let email = email_of(&first);
|
||||
assert_eq!(email.fetched, 9, "all 9 synthetic messages are new");
|
||||
assert!(email.bytes > 0, "new mail is sized: {}", email.bytes);
|
||||
assert_eq!(email.failed, 0);
|
||||
|
||||
// After a real import, a re-estimate finds nothing new.
|
||||
run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap();
|
||||
let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
|
||||
assert_eq!(email_of(&again).fetched, 0, "nothing new after importing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_mapping_produces_expected_mailbox_tree() {
|
||||
let td = TempDir::new().unwrap();
|
||||
|
||||
Reference in new issue
Block a user