estimate: itemize a Takeout's calendars and contacts too #71

Merged
jcoffey-dev merged 1 commits from feat/estimate-takeout-pim into main 2026-11-12 17:31:08 +00:00
4 changed files with 183 additions and 34 deletions

No files matched your search

+3 -2
View File
@@ -39,8 +39,9 @@ takes `--dry-run` and then only reports the plan. An `import` also takes
`--estimate`, which connects to the source and reports how much is new to
fetch — counts, bytes and a rough time — before committing, and writes
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange
(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall
back to a plain dry-run. `inspect` reads an archive and changes nothing.
(`ews`) and a `takeout` (whose files it reads to count the mail, events and
contacts that are new); other sources fall back to a plain dry-run. `inspect`
reads an archive and changes nothing.
The archive is also a backup. Import on a schedule, keep the file, and
restore it later with `export`. An archive remembers which account filled it
+5 -5
View File
@@ -156,9 +156,9 @@ struct GlobalArgs {
help = "Preview an import: counts, sizes and a rough time, before any writes",
long_help = "Preview an import before committing: for each type, how much is new since \
the archive was last filled, how many bytes that is, and a rough time.\n \
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \
today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \
never writes. On an export it is just --dry-run."
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \
events and contacts) today; other sources fall back to --dry-run with a note. It \
implies --dry-run, so it never writes. On an export it is just --dry-run."
)]
estimate: bool,
@@ -930,8 +930,8 @@ impl Action {
// through the same command but report differently, so they stay
// on the fallback for now.
Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav),
// Takeout reads the mbox files to count mail; its calendars and
// contacts are not itemized yet.
// Takeout reads its .mbox, .ics and .vcf files to count the mail,
// events and contacts that are new, and sizes each.
Action::ImportTakeout(c, _) => c.estimate,
// EWS FindItem returns item:Size, so mail, calendars and contacts
// are all counted and sized from the enumeration.
+137 -20
View File
@@ -25,6 +25,7 @@ use super::labels::MappingOptions;
use super::mail;
use super::mbox::MessageIterator;
use super::walk::{self, FileKind};
use crate::sync::import_dav::calcard;
#[derive(Debug)]
pub struct TakeoutImportConfig {
@@ -366,29 +367,54 @@ fn dry_run_summary(
" existing in archive: emails={local_emails} events={local_events} cards={local_cards}"
);
}
// --estimate: a plain dry run counts files; an estimate reads the mbox
// files to count the messages that are actually new and sum their bytes.
// Calendars and contacts are not itemized yet -- the file counts above
// stand in for them.
let email = if want_sizes {
estimate_mail(conn, source_id, walk_result, logger)
// --estimate: a plain dry run counts files; an estimate reads them to count
// the objects that are actually new since the archive was last filled and
// sum their bytes -- messages from the mboxes, events from the .ics files,
// cards from the .vcf files -- deduplicated by the same key the import uses.
let (email, events, cards) = if want_sizes {
(
estimate_mail(conn, source_id, walk_result, logger),
estimate_calendar(conn, source_id, walk_result, logger),
estimate_contacts(conn, source_id, walk_result, logger),
)
} else {
TypeCounts::default()
Default::default()
};
Summary {
per_type: vec![
("mailbox", TypeCounts::default()),
("email", email),
("calendar", TypeCounts::default()),
("calendarevent", TypeCounts::default()),
("calendarevent", events),
("addressbook", TypeCounts::default()),
("contactcard", TypeCounts::default()),
("contactcard", cards),
],
retries_observed: 0,
retry_after_sleeps: 0,
}
}
/// Loads the external ids (content hashes, event/card UIDs) the archive already
/// holds for `type_name`, as the dedup set an estimate scans against. An empty
/// set on a fresh or unreadable archive just means everything reads as new.
fn archived_keys(
conn: &Connection,
source_id: i64,
type_name: &str,
logger: Logger,
) -> HashSet<String> {
if source_id < 0 {
return HashSet::new();
}
match takeout_ids::all_for_type(conn, source_id, type_name) {
Ok(m) => m.into_keys().collect(),
Err(e) => {
logger.warn(&format!("dry-run: reading archived {type_name} ids: {e}"));
HashSet::new()
}
}
}
/// Scans the mbox files for `--estimate`: counts the messages not already in
/// the archive and sums their bytes, deduplicating by the same content hash
/// the import uses, so a message that appears in two mboxes counts once. It
@@ -401,17 +427,7 @@ fn estimate_mail(
logger: Logger,
) -> TypeCounts {
let mut email = TypeCounts::default();
let mut seen: HashSet<String> = if source_id < 0 {
HashSet::new()
} else {
match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) {
Ok(m) => m.into_keys().collect(),
Err(e) => {
logger.warn(&format!("dry-run: reading archived message ids: {e}"));
HashSet::new()
}
}
};
let mut seen = archived_keys(conn, source_id, takeout_ids::EMAIL, logger);
for file in walk_result.by_kind(FileKind::Mbox) {
let reader = match File::open(&file.path) {
Ok(f) => BufReader::new(f),
@@ -448,6 +464,107 @@ fn estimate_mail(
email
}
/// Scans the `.ics` files for `--estimate`: counts the events not already in
/// the archive and sums the bytes of the JSCalendar record each would store.
/// It derives each event's UID exactly as the import does -- the event's own
/// UID, or one synthesised from the file path and ordinal when it has none --
/// so the same event in two files, or one already imported, counts once. It
/// parses the iCalendar but writes nothing.
fn estimate_calendar(
conn: &Connection,
source_id: i64,
walk_result: &walk::WalkResult,
logger: Logger,
) -> TypeCounts {
let mut events = TypeCounts::default();
let mut seen = archived_keys(conn, source_id, takeout_ids::CALENDAR_EVENT, logger);
for file in walk_result.by_kind(FileKind::Ics) {
let text = match std::fs::read_to_string(&file.path) {
Ok(s) => s,
Err(e) => {
logger.warn(&format!("dry-run read {:?}: {e}", file.path));
events.failed += 1;
continue;
}
};
let entries = match calcard::ical_to_jscalendar_entries(&text) {
Ok(v) => v,
Err(calcard::CalcardError::NoEntries) => continue,
Err(e) => {
logger.warn(&format!("dry-run {:?}: iCalendar parse: {e}", file.path));
events.failed += 1;
continue;
}
};
for (ordinal, mut entry) in entries.into_iter().enumerate() {
let (_is_draft, _use_default_alerts, uid_field) =
calcard::strip_extracted_fields_from_event(&mut entry.data);
let uid = match uid_field
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
Some(u) => u.to_owned(),
None => {
calcard::synthesise_uid(&format!("{}#{ordinal}", file.path.to_string_lossy()))
}
};
let size = entry.data.to_string().len() as u64;
if seen.insert(uid) {
events.created += 1;
events.fetched += 1;
events.bytes += size;
} else {
events.skipped += 1;
}
}
}
events
}
/// Scans the `.vcf` files for `--estimate`: counts the contacts not already in
/// the archive and sums the bytes of the JSContact record each would store,
/// deduplicating by the card's UID, the same key the import keeps. It parses
/// the vCards but writes nothing.
fn estimate_contacts(
conn: &Connection,
source_id: i64,
walk_result: &walk::WalkResult,
logger: Logger,
) -> TypeCounts {
let mut cards = TypeCounts::default();
let mut seen = archived_keys(conn, source_id, takeout_ids::CONTACT_CARD, logger);
for file in walk_result.by_kind(FileKind::Vcf) {
let text = match std::fs::read_to_string(&file.path) {
Ok(s) => s,
Err(e) => {
logger.warn(&format!("dry-run read {:?}: {e}", file.path));
cards.failed += 1;
continue;
}
};
for chunk in contacts::split_vcards(&text) {
match calcard::vcard_to_jscontact(&chunk, &chunk) {
Ok(card) => {
let size = card.data.to_string().len() as u64;
if seen.insert(card.uid) {
cards.created += 1;
cards.fetched += 1;
cards.bytes += size;
} else {
cards.skipped += 1;
}
}
Err(e) => {
logger.warn(&format!("dry-run {:?}: vCard parse: {e}", file.path));
cards.failed += 1;
}
}
}
}
cards
}
fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 {
conn.query_row(
"SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2",
+38 -7
View File
@@ -282,7 +282,7 @@ fn synthetic_takeout_imports_expected_row_counts() {
}
#[test]
fn estimate_counts_new_mail_and_sizes_it() {
fn estimate_counts_new_objects_and_sizes_them() {
let td = TempDir::new().unwrap();
SyntheticTakeout::build(td.path());
let archive = td.path().join("a.sqlite");
@@ -293,27 +293,58 @@ fn estimate_counts_new_mail_and_sizes_it() {
c.estimate = true;
c
};
let email_of = |summary: &inbuxa_migrate::sync::Summary| {
let by = |summary: &inbuxa_migrate::sync::Summary, t: &str| {
summary
.per_type
.iter()
.find(|(n, _)| *n == "email")
.find(|(n, _)| *n == t)
.unwrap()
.1
.clone()
};
// A fresh archive: every synthetic message is new, and it is sized.
// A fresh archive: every synthetic object is new, across all three kinds,
// and each kind is sized.
let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
let email = email_of(&first);
let email = by(&first, "email");
assert_eq!(email.fetched, 9, "all 9 synthetic messages are new");
assert!(email.bytes > 0, "new mail is sized: {}", email.bytes);
assert_eq!(email.failed, 0);
// After a real import, a re-estimate finds nothing new.
let events = by(&first, "calendarevent");
assert!(events.fetched >= 2, "the .ics events are new: {events:?}");
assert!(events.bytes > 0, "new events are sized: {}", events.bytes);
assert_eq!(events.failed, 0);
// The two .vcf files carry the same cards; UID dedup collapses them to the
// four the import creates, so the estimate must report four, not eight.
let cards = by(&first, "contactcard");
assert_eq!(
cards.fetched, 4,
"cards dedup by UID across the two .vcf files"
);
assert!(cards.bytes > 0, "new cards are sized: {}", cards.bytes);
assert_eq!(cards.failed, 0);
// After a real import, a re-estimate finds nothing new of any kind.
run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap();
let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
assert_eq!(email_of(&again).fetched, 0, "nothing new after importing");
assert_eq!(
by(&again, "email").fetched,
0,
"no new mail after importing"
);
assert_eq!(
by(&again, "calendarevent").fetched,
0,
"no new events after importing"
);
assert_eq!(
by(&again, "contactcard").fetched,
0,
"no new cards after importing"
);
}
#[test]