estimate: itemize a Takeout's calendars and contacts too #71
No files matched your search
@@ -39,8 +39,9 @@ takes `--dry-run` and then only reports the plan. An `import` also takes
|
||||
`--estimate`, which connects to the source and reports how much is new to
|
||||
fetch — counts, bytes and a rough time — before committing, and writes
|
||||
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange
|
||||
(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall
|
||||
back to a plain dry-run. `inspect` reads an archive and changes nothing.
|
||||
(`ews`) and a `takeout` (whose files it reads to count the mail, events and
|
||||
contacts that are new); other sources fall back to a plain dry-run. `inspect`
|
||||
reads an archive and changes nothing.
|
||||
|
||||
The archive is also a backup. Import on a schedule, keep the file, and
|
||||
restore it later with `export`. An archive remembers which account filled it
|
||||
|
||||
+5
-5
@@ -156,9 +156,9 @@ struct GlobalArgs {
|
||||
help = "Preview an import: counts, sizes and a rough time, before any writes",
|
||||
long_help = "Preview an import before committing: for each type, how much is new since \
|
||||
the archive was last filled, how many bytes that is, and a rough time.\n \
|
||||
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \
|
||||
today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \
|
||||
never writes. On an export it is just --dry-run."
|
||||
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \
|
||||
events and contacts) today; other sources fall back to --dry-run with a note. It \
|
||||
implies --dry-run, so it never writes. On an export it is just --dry-run."
|
||||
)]
|
||||
estimate: bool,
|
||||
|
||||
@@ -930,8 +930,8 @@ impl Action {
|
||||
// through the same command but report differently, so they stay
|
||||
// on the fallback for now.
|
||||
Action::ImportDav(c, cfg) => c.estimate && !matches!(cfg.kind, DavKindArg::Webdav),
|
||||
// Takeout reads the mbox files to count mail; its calendars and
|
||||
// contacts are not itemized yet.
|
||||
// Takeout reads its .mbox, .ics and .vcf files to count the mail,
|
||||
// events and contacts that are new, and sizes each.
|
||||
Action::ImportTakeout(c, _) => c.estimate,
|
||||
// EWS FindItem returns item:Size, so mail, calendars and contacts
|
||||
// are all counted and sized from the enumeration.
|
||||
|
||||
@@ -25,6 +25,7 @@ use super::labels::MappingOptions;
|
||||
use super::mail;
|
||||
use super::mbox::MessageIterator;
|
||||
use super::walk::{self, FileKind};
|
||||
use crate::sync::import_dav::calcard;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TakeoutImportConfig {
|
||||
@@ -366,29 +367,54 @@ fn dry_run_summary(
|
||||
" existing in archive: emails={local_emails} events={local_events} cards={local_cards}"
|
||||
);
|
||||
}
|
||||
// --estimate: a plain dry run counts files; an estimate reads the mbox
|
||||
// files to count the messages that are actually new and sum their bytes.
|
||||
// Calendars and contacts are not itemized yet -- the file counts above
|
||||
// stand in for them.
|
||||
let email = if want_sizes {
|
||||
estimate_mail(conn, source_id, walk_result, logger)
|
||||
// --estimate: a plain dry run counts files; an estimate reads them to count
|
||||
// the objects that are actually new since the archive was last filled and
|
||||
// sum their bytes -- messages from the mboxes, events from the .ics files,
|
||||
// cards from the .vcf files -- deduplicated by the same key the import uses.
|
||||
let (email, events, cards) = if want_sizes {
|
||||
(
|
||||
estimate_mail(conn, source_id, walk_result, logger),
|
||||
estimate_calendar(conn, source_id, walk_result, logger),
|
||||
estimate_contacts(conn, source_id, walk_result, logger),
|
||||
)
|
||||
} else {
|
||||
TypeCounts::default()
|
||||
Default::default()
|
||||
};
|
||||
Summary {
|
||||
per_type: vec![
|
||||
("mailbox", TypeCounts::default()),
|
||||
("email", email),
|
||||
("calendar", TypeCounts::default()),
|
||||
("calendarevent", TypeCounts::default()),
|
||||
("calendarevent", events),
|
||||
("addressbook", TypeCounts::default()),
|
||||
("contactcard", TypeCounts::default()),
|
||||
("contactcard", cards),
|
||||
],
|
||||
retries_observed: 0,
|
||||
retry_after_sleeps: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the external ids (content hashes, event/card UIDs) the archive already
|
||||
/// holds for `type_name`, as the dedup set an estimate scans against. An empty
|
||||
/// set on a fresh or unreadable archive just means everything reads as new.
|
||||
fn archived_keys(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
type_name: &str,
|
||||
logger: Logger,
|
||||
) -> HashSet<String> {
|
||||
if source_id < 0 {
|
||||
return HashSet::new();
|
||||
}
|
||||
match takeout_ids::all_for_type(conn, source_id, type_name) {
|
||||
Ok(m) => m.into_keys().collect(),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run: reading archived {type_name} ids: {e}"));
|
||||
HashSet::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scans the mbox files for `--estimate`: counts the messages not already in
|
||||
/// the archive and sums their bytes, deduplicating by the same content hash
|
||||
/// the import uses, so a message that appears in two mboxes counts once. It
|
||||
@@ -401,17 +427,7 @@ fn estimate_mail(
|
||||
logger: Logger,
|
||||
) -> TypeCounts {
|
||||
let mut email = TypeCounts::default();
|
||||
let mut seen: HashSet<String> = if source_id < 0 {
|
||||
HashSet::new()
|
||||
} else {
|
||||
match takeout_ids::all_for_type(conn, source_id, takeout_ids::EMAIL) {
|
||||
Ok(m) => m.into_keys().collect(),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run: reading archived message ids: {e}"));
|
||||
HashSet::new()
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut seen = archived_keys(conn, source_id, takeout_ids::EMAIL, logger);
|
||||
for file in walk_result.by_kind(FileKind::Mbox) {
|
||||
let reader = match File::open(&file.path) {
|
||||
Ok(f) => BufReader::new(f),
|
||||
@@ -448,6 +464,107 @@ fn estimate_mail(
|
||||
email
|
||||
}
|
||||
|
||||
/// Scans the `.ics` files for `--estimate`: counts the events not already in
|
||||
/// the archive and sums the bytes of the JSCalendar record each would store.
|
||||
/// It derives each event's UID exactly as the import does -- the event's own
|
||||
/// UID, or one synthesised from the file path and ordinal when it has none --
|
||||
/// so the same event in two files, or one already imported, counts once. It
|
||||
/// parses the iCalendar but writes nothing.
|
||||
fn estimate_calendar(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
walk_result: &walk::WalkResult,
|
||||
logger: Logger,
|
||||
) -> TypeCounts {
|
||||
let mut events = TypeCounts::default();
|
||||
let mut seen = archived_keys(conn, source_id, takeout_ids::CALENDAR_EVENT, logger);
|
||||
for file in walk_result.by_kind(FileKind::Ics) {
|
||||
let text = match std::fs::read_to_string(&file.path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run read {:?}: {e}", file.path));
|
||||
events.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let entries = match calcard::ical_to_jscalendar_entries(&text) {
|
||||
Ok(v) => v,
|
||||
Err(calcard::CalcardError::NoEntries) => continue,
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run {:?}: iCalendar parse: {e}", file.path));
|
||||
events.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for (ordinal, mut entry) in entries.into_iter().enumerate() {
|
||||
let (_is_draft, _use_default_alerts, uid_field) =
|
||||
calcard::strip_extracted_fields_from_event(&mut entry.data);
|
||||
let uid = match uid_field
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
Some(u) => u.to_owned(),
|
||||
None => {
|
||||
calcard::synthesise_uid(&format!("{}#{ordinal}", file.path.to_string_lossy()))
|
||||
}
|
||||
};
|
||||
let size = entry.data.to_string().len() as u64;
|
||||
if seen.insert(uid) {
|
||||
events.created += 1;
|
||||
events.fetched += 1;
|
||||
events.bytes += size;
|
||||
} else {
|
||||
events.skipped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
events
|
||||
}
|
||||
|
||||
/// Scans the `.vcf` files for `--estimate`: counts the contacts not already in
|
||||
/// the archive and sums the bytes of the JSContact record each would store,
|
||||
/// deduplicating by the card's UID, the same key the import keeps. It parses
|
||||
/// the vCards but writes nothing.
|
||||
fn estimate_contacts(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
walk_result: &walk::WalkResult,
|
||||
logger: Logger,
|
||||
) -> TypeCounts {
|
||||
let mut cards = TypeCounts::default();
|
||||
let mut seen = archived_keys(conn, source_id, takeout_ids::CONTACT_CARD, logger);
|
||||
for file in walk_result.by_kind(FileKind::Vcf) {
|
||||
let text = match std::fs::read_to_string(&file.path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run read {:?}: {e}", file.path));
|
||||
cards.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for chunk in contacts::split_vcards(&text) {
|
||||
match calcard::vcard_to_jscontact(&chunk, &chunk) {
|
||||
Ok(card) => {
|
||||
let size = card.data.to_string().len() as u64;
|
||||
if seen.insert(card.uid) {
|
||||
cards.created += 1;
|
||||
cards.fetched += 1;
|
||||
cards.bytes += size;
|
||||
} else {
|
||||
cards.skipped += 1;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.warn(&format!("dry-run {:?}: vCard parse: {e}", file.path));
|
||||
cards.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cards
|
||||
}
|
||||
|
||||
fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 {
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2",
|
||||
|
||||
+38
-7
@@ -282,7 +282,7 @@ fn synthetic_takeout_imports_expected_row_counts() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_counts_new_mail_and_sizes_it() {
|
||||
fn estimate_counts_new_objects_and_sizes_them() {
|
||||
let td = TempDir::new().unwrap();
|
||||
SyntheticTakeout::build(td.path());
|
||||
let archive = td.path().join("a.sqlite");
|
||||
@@ -293,27 +293,58 @@ fn estimate_counts_new_mail_and_sizes_it() {
|
||||
c.estimate = true;
|
||||
c
|
||||
};
|
||||
let email_of = |summary: &inbuxa_migrate::sync::Summary| {
|
||||
let by = |summary: &inbuxa_migrate::sync::Summary, t: &str| {
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(n, _)| *n == "email")
|
||||
.find(|(n, _)| *n == t)
|
||||
.unwrap()
|
||||
.1
|
||||
.clone()
|
||||
};
|
||||
|
||||
// A fresh archive: every synthetic message is new, and it is sized.
|
||||
// A fresh archive: every synthetic object is new, across all three kinds,
|
||||
// and each kind is sized.
|
||||
let first = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
|
||||
let email = email_of(&first);
|
||||
|
||||
let email = by(&first, "email");
|
||||
assert_eq!(email.fetched, 9, "all 9 synthetic messages are new");
|
||||
assert!(email.bytes > 0, "new mail is sized: {}", email.bytes);
|
||||
assert_eq!(email.failed, 0);
|
||||
|
||||
// After a real import, a re-estimate finds nothing new.
|
||||
let events = by(&first, "calendarevent");
|
||||
assert!(events.fetched >= 2, "the .ics events are new: {events:?}");
|
||||
assert!(events.bytes > 0, "new events are sized: {}", events.bytes);
|
||||
assert_eq!(events.failed, 0);
|
||||
|
||||
// The two .vcf files carry the same cards; UID dedup collapses them to the
|
||||
// four the import creates, so the estimate must report four, not eight.
|
||||
let cards = by(&first, "contactcard");
|
||||
assert_eq!(
|
||||
cards.fetched, 4,
|
||||
"cards dedup by UID across the two .vcf files"
|
||||
);
|
||||
assert!(cards.bytes > 0, "new cards are sized: {}", cards.bytes);
|
||||
assert_eq!(cards.failed, 0);
|
||||
|
||||
// After a real import, a re-estimate finds nothing new of any kind.
|
||||
run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap();
|
||||
let again = run(estimate_cfg(&archive), config_for(td.path().to_path_buf())).unwrap();
|
||||
assert_eq!(email_of(&again).fetched, 0, "nothing new after importing");
|
||||
assert_eq!(
|
||||
by(&again, "email").fetched,
|
||||
0,
|
||||
"no new mail after importing"
|
||||
);
|
||||
assert_eq!(
|
||||
by(&again, "calendarevent").fetched,
|
||||
0,
|
||||
"no new events after importing"
|
||||
);
|
||||
assert_eq!(
|
||||
by(&again, "contactcard").fetched,
|
||||
0,
|
||||
"no new cards after importing"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in new issue
Block a user