estimate: size an Exchange (EWS) mailbox #70

Merged
jcoffey-dev merged 1 commits from feat/estimate-ews into main 2026-11-12 16:45:23 +00:00
6 changed files with 171 additions and 9 deletions

No files matched your search

+3 -3
View File
@@ -38,9 +38,9 @@ later `import` into the same archive adds what arrived since. Every command
takes `--dry-run` and then only reports the plan. An `import` also takes
`--estimate`, which connects to the source and reports how much is new to
fetch — counts, bytes and a rough time — before committing, and writes
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav` and a
`takeout`'s mail (which it reads to count); other sources fall back to a plain
dry-run. `inspect` reads an archive and changes nothing.
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange
(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall
back to a plain dry-run. `inspect` reads an archive and changes nothing.
The archive is also a backup. Import on a schedule, keep the file, and
restore it later with `export`. An archive remembers which account filled it
+6 -3
View File
@@ -156,9 +156,9 @@ struct GlobalArgs {
help = "Preview an import: counts, sizes and a rough time, before any writes",
long_help = "Preview an import before committing: for each type, how much is new since \
the archive was last filled, how many bytes that is, and a rough time.\n \
Available for imap, maildir, caldav, carddav and takeout (its mail) today; other \
sources fall back to --dry-run with a note. It implies --dry-run, so it never writes. \
On an export it is just --dry-run."
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \
today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \
never writes. On an export it is just --dry-run."
)]
estimate: bool,
@@ -933,6 +933,9 @@ impl Action {
// Takeout reads the mbox files to count mail; its calendars and
// contacts are not itemized yet.
Action::ImportTakeout(c, _) => c.estimate,
// EWS FindItem returns item:Size, so mail, calendars and contacts
// are all counted and sized from the enumeration.
Action::ImportExchangeEws(c, _) => c.estimate,
_ => false,
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ fn run() -> i32 {
if action.estimate_requested() && !do_estimate {
eprintln!(
"note: --estimate isn't available for this source yet; running --dry-run instead \
(its plan is below). Estimates work for imap, maildir, caldav, carddav and takeout today."
(its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS) and takeout today."
);
}
let mut quiet_report = false;
+74 -2
View File
@@ -19,7 +19,9 @@ use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS};
use crate::sync::{CommonConfig, Summary, TypeCounts};
use super::folders::{self, plan_folders};
use super::items::{self, ItemRunCtx};
use super::{calendar, contacts, messages};
use crate::db::exchange_ews_ids;
use crate::net::CertOverride;
#[derive(Debug, Clone)]
@@ -102,7 +104,16 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
};
if common.dry_run {
let summary = run_dry(&conn, &client, &session_url, &config, logger)?;
let source_id = db::sources::find_source(&conn, &key)?.unwrap_or(-1);
let summary = run_dry(
&conn,
&client,
&session_url,
&config,
source_id,
common.estimate,
logger,
)?;
return Ok(summary);
}
@@ -188,10 +199,12 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
}
fn run_dry(
_conn: &Connection,
conn: &Connection,
client: &EwsClient,
session_url: &str,
config: &EwsImportConfig,
source_id: i64,
want_sizes: bool,
logger: crate::logging::Logger,
) -> Result<Summary, Error> {
let plan = plan_folders(
@@ -218,9 +231,68 @@ fn run_dry(
summary.per_type.push(("mailbox", mailbox_counts));
summary.per_type.push(("calendar", calendar_counts));
summary.per_type.push(("addressbook", addressbook_counts));
// --estimate: enumerate each folder's items (FindItem gives item:Size, so
// this never fetches a body) and count and size what is new against the
// archive. A plain dry run stops at the folder counts above.
if want_sizes {
let ctx = ItemRunCtx {
client,
url: session_url,
source_id,
batch_size: config.getitem_batch.max(1),
batch_bytes: config.getitem_batch_bytes.max(1),
attachment_batch: config.attachment_batch.max(1),
connections: config.ews_connections.clamp(1, 8),
// FindItem (not SyncFolderItems) so every enumeration carries sizes.
use_syncfolderitems: false,
sync_batch: items::SYNC_BATCH_MAX,
logger,
};
let email = estimate_items(conn, &ctx, &plan.mail, exchange_ews_ids::EMAIL);
let events = estimate_items(conn, &ctx, &plan.calendar, exchange_ews_ids::CALENDAR_EVENT);
let cards = estimate_items(conn, &ctx, &plan.contacts, exchange_ews_ids::CONTACT_CARD);
summary.per_type.push(("email", email));
summary.per_type.push(("calendarevent", events));
summary.per_type.push(("contactcard", cards));
}
Ok(summary)
}
/// Counts and sizes the items in `folders` that are new against the archive,
/// for `--estimate`. FindItem returns `item:Size`, so this reads no bodies and
/// writes nothing; the diff is the same one the import uses, so a re-estimate
/// against a part-filled archive reports only what is left.
fn estimate_items(
conn: &Connection,
ctx: &ItemRunCtx<'_>,
folders: &[folders::ClassifiedFolder],
item_type: &str,
) -> TypeCounts {
let mut counts = TypeCounts::default();
for cf in folders {
let fid = &cf.folder.folder_id;
let outcome = match items::enumerate_folder(ctx, fid, None) {
Ok(o) => o,
Err(e) => {
ctx.logger.warn(&format!(
"dry-run enumerate {:?}: {e}",
cf.folder.display_name
));
counts.failed += 1;
continue;
}
};
let local = exchange_ews_ids::items_in_folder(conn, ctx.source_id, item_type, &fid.id)
.unwrap_or_default();
let (new, bytes) = items::new_count_and_bytes(&outcome, &local);
counts.created += new;
counts.fetched += new;
counts.bytes += bytes;
}
counts
}
fn resolve_auth(auth: &EwsAuth) -> Result<(Auth, Option<AcquiredToken>), Error> {
match auth {
EwsAuth::Basic { user, password } => Ok((
+19
View File
@@ -221,6 +221,25 @@ pub fn plan_for(
}
}
/// Count the new items an enumeration turned up and sum the sizes the server
/// reported for them. Sizes come from FindItem's `item:Size`; an item the
/// server returned without a size still counts, it just adds no bytes. Used by
/// `--estimate`, which enumerates with FindItem precisely so the sizes are
/// present.
pub fn new_count_and_bytes(
outcome: &EnumerationOutcome,
local: &[crate::db::exchange_ews_ids::ItemRow],
) -> (u64, u64) {
let plan = plan_for(outcome, local);
let count = plan.new.len() as u64;
let bytes = plan
.new
.iter()
.filter_map(|id| outcome.sizes.get(&id.id).copied())
.sum();
(count, bytes)
}
fn diff_delta(
server_changes: &[EnumeratedItem],
deletions: &[String],
+68
View File
@@ -534,6 +534,74 @@ fn coordinator_diff_classifies_new_vanished_changed_unchanged() {
assert_eq!(plan.vanished[0].0, "Z");
}
#[test]
fn estimate_counts_new_items_and_sums_their_sizes() {
use inbuxa_migrate::db::exchange_ews_ids::ItemRow;
use inbuxa_migrate::sync::import_exchange_ews::items::{
EnumeratedItem, EnumerationMode, EnumerationOutcome, new_count_and_bytes,
};
use std::collections::HashMap;
// Three items on the server; A is already local (not new), B and C are new.
let items = vec![
EnumeratedItem {
element: "Message".into(),
id: ItemId::new("A", "ck-1"),
},
EnumeratedItem {
element: "Message".into(),
id: ItemId::new("B", "ck-1"),
},
EnumeratedItem {
element: "Message".into(),
id: ItemId::new("C", "ck-1"),
},
];
let mut sizes = HashMap::new();
sizes.insert("A".to_string(), 1000u64);
sizes.insert("B".to_string(), 2048u64);
// C has no reported size: it still counts, but adds no bytes.
let outcome = EnumerationOutcome {
items,
mode: EnumerationMode::Full,
sizes,
};
let local = vec![ItemRow {
item_id: "A".into(),
change_key: "ck-1".into(),
local_id: 1,
}];
let (count, bytes) = new_count_and_bytes(&outcome, &local);
assert_eq!(count, 2, "B and C are new; A is already local");
assert_eq!(
bytes, 2048,
"only B's size is known; C has none; A is not new"
);
// Nothing new once everything is local: no count, no bytes.
let all_local = vec![
ItemRow {
item_id: "A".into(),
change_key: "ck-1".into(),
local_id: 1,
},
ItemRow {
item_id: "B".into(),
change_key: "ck-1".into(),
local_id: 2,
},
ItemRow {
item_id: "C".into(),
change_key: "ck-1".into(),
local_id: 3,
},
];
let (count, bytes) = new_count_and_bytes(&outcome, &all_local);
assert_eq!(count, 0);
assert_eq!(bytes, 0);
}
#[test]
fn source_change_protection_refuses_different_account() {
use inbuxa_migrate::db;