estimate: size an Exchange (EWS) mailbox #70
No files matched your search
@@ -38,9 +38,9 @@ later `import` into the same archive adds what arrived since. Every command
|
||||
takes `--dry-run` and then only reports the plan. An `import` also takes
|
||||
`--estimate`, which connects to the source and reports how much is new to
|
||||
fetch — counts, bytes and a rough time — before committing, and writes
|
||||
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav` and a
|
||||
`takeout`'s mail (which it reads to count); other sources fall back to a plain
|
||||
dry-run. `inspect` reads an archive and changes nothing.
|
||||
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange
|
||||
(`ews`) and a `takeout`'s mail (which it reads to count); other sources fall
|
||||
back to a plain dry-run. `inspect` reads an archive and changes nothing.
|
||||
|
||||
The archive is also a backup. Import on a schedule, keep the file, and
|
||||
restore it later with `export`. An archive remembers which account filled it
|
||||
|
||||
+6
-3
@@ -156,9 +156,9 @@ struct GlobalArgs {
|
||||
help = "Preview an import: counts, sizes and a rough time, before any writes",
|
||||
long_help = "Preview an import before committing: for each type, how much is new since \
|
||||
the archive was last filled, how many bytes that is, and a rough time.\n \
|
||||
Available for imap, maildir, caldav, carddav and takeout (its mail) today; other \
|
||||
sources fall back to --dry-run with a note. It implies --dry-run, so it never writes. \
|
||||
On an export it is just --dry-run."
|
||||
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (its mail) \
|
||||
today; other sources fall back to --dry-run with a note. It implies --dry-run, so it \
|
||||
never writes. On an export it is just --dry-run."
|
||||
)]
|
||||
estimate: bool,
|
||||
|
||||
@@ -933,6 +933,9 @@ impl Action {
|
||||
// Takeout reads the mbox files to count mail; its calendars and
|
||||
// contacts are not itemized yet.
|
||||
Action::ImportTakeout(c, _) => c.estimate,
|
||||
// EWS FindItem returns item:Size, so mail, calendars and contacts
|
||||
// are all counted and sized from the enumeration.
|
||||
Action::ImportExchangeEws(c, _) => c.estimate,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ fn run() -> i32 {
|
||||
if action.estimate_requested() && !do_estimate {
|
||||
eprintln!(
|
||||
"note: --estimate isn't available for this source yet; running --dry-run instead \
|
||||
(its plan is below). Estimates work for imap, maildir, caldav, carddav and takeout today."
|
||||
(its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS) and takeout today."
|
||||
);
|
||||
}
|
||||
let mut quiet_report = false;
|
||||
|
||||
@@ -19,7 +19,9 @@ use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS};
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
|
||||
use super::folders::{self, plan_folders};
|
||||
use super::items::{self, ItemRunCtx};
|
||||
use super::{calendar, contacts, messages};
|
||||
use crate::db::exchange_ews_ids;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -102,7 +104,16 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
};
|
||||
|
||||
if common.dry_run {
|
||||
let summary = run_dry(&conn, &client, &session_url, &config, logger)?;
|
||||
let source_id = db::sources::find_source(&conn, &key)?.unwrap_or(-1);
|
||||
let summary = run_dry(
|
||||
&conn,
|
||||
&client,
|
||||
&session_url,
|
||||
&config,
|
||||
source_id,
|
||||
common.estimate,
|
||||
logger,
|
||||
)?;
|
||||
return Ok(summary);
|
||||
}
|
||||
|
||||
@@ -188,10 +199,12 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
}
|
||||
|
||||
fn run_dry(
|
||||
_conn: &Connection,
|
||||
conn: &Connection,
|
||||
client: &EwsClient,
|
||||
session_url: &str,
|
||||
config: &EwsImportConfig,
|
||||
source_id: i64,
|
||||
want_sizes: bool,
|
||||
logger: crate::logging::Logger,
|
||||
) -> Result<Summary, Error> {
|
||||
let plan = plan_folders(
|
||||
@@ -218,9 +231,68 @@ fn run_dry(
|
||||
summary.per_type.push(("mailbox", mailbox_counts));
|
||||
summary.per_type.push(("calendar", calendar_counts));
|
||||
summary.per_type.push(("addressbook", addressbook_counts));
|
||||
|
||||
// --estimate: enumerate each folder's items (FindItem gives item:Size, so
|
||||
// this never fetches a body) and count and size what is new against the
|
||||
// archive. A plain dry run stops at the folder counts above.
|
||||
if want_sizes {
|
||||
let ctx = ItemRunCtx {
|
||||
client,
|
||||
url: session_url,
|
||||
source_id,
|
||||
batch_size: config.getitem_batch.max(1),
|
||||
batch_bytes: config.getitem_batch_bytes.max(1),
|
||||
attachment_batch: config.attachment_batch.max(1),
|
||||
connections: config.ews_connections.clamp(1, 8),
|
||||
// FindItem (not SyncFolderItems) so every enumeration carries sizes.
|
||||
use_syncfolderitems: false,
|
||||
sync_batch: items::SYNC_BATCH_MAX,
|
||||
logger,
|
||||
};
|
||||
let email = estimate_items(conn, &ctx, &plan.mail, exchange_ews_ids::EMAIL);
|
||||
let events = estimate_items(conn, &ctx, &plan.calendar, exchange_ews_ids::CALENDAR_EVENT);
|
||||
let cards = estimate_items(conn, &ctx, &plan.contacts, exchange_ews_ids::CONTACT_CARD);
|
||||
summary.per_type.push(("email", email));
|
||||
summary.per_type.push(("calendarevent", events));
|
||||
summary.per_type.push(("contactcard", cards));
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Counts and sizes the items in `folders` that are new against the archive,
|
||||
/// for `--estimate`. FindItem returns `item:Size`, so this reads no bodies and
|
||||
/// writes nothing; the diff is the same one the import uses, so a re-estimate
|
||||
/// against a part-filled archive reports only what is left.
|
||||
fn estimate_items(
|
||||
conn: &Connection,
|
||||
ctx: &ItemRunCtx<'_>,
|
||||
folders: &[folders::ClassifiedFolder],
|
||||
item_type: &str,
|
||||
) -> TypeCounts {
|
||||
let mut counts = TypeCounts::default();
|
||||
for cf in folders {
|
||||
let fid = &cf.folder.folder_id;
|
||||
let outcome = match items::enumerate_folder(ctx, fid, None) {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
ctx.logger.warn(&format!(
|
||||
"dry-run enumerate {:?}: {e}",
|
||||
cf.folder.display_name
|
||||
));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let local = exchange_ews_ids::items_in_folder(conn, ctx.source_id, item_type, &fid.id)
|
||||
.unwrap_or_default();
|
||||
let (new, bytes) = items::new_count_and_bytes(&outcome, &local);
|
||||
counts.created += new;
|
||||
counts.fetched += new;
|
||||
counts.bytes += bytes;
|
||||
}
|
||||
counts
|
||||
}
|
||||
|
||||
fn resolve_auth(auth: &EwsAuth) -> Result<(Auth, Option<AcquiredToken>), Error> {
|
||||
match auth {
|
||||
EwsAuth::Basic { user, password } => Ok((
|
||||
|
||||
@@ -221,6 +221,25 @@ pub fn plan_for(
|
||||
}
|
||||
}
|
||||
|
||||
/// Count the new items an enumeration turned up and sum the sizes the server
|
||||
/// reported for them. Sizes come from FindItem's `item:Size`; an item the
|
||||
/// server returned without a size still counts, it just adds no bytes. Used by
|
||||
/// `--estimate`, which enumerates with FindItem precisely so the sizes are
|
||||
/// present.
|
||||
pub fn new_count_and_bytes(
|
||||
outcome: &EnumerationOutcome,
|
||||
local: &[crate::db::exchange_ews_ids::ItemRow],
|
||||
) -> (u64, u64) {
|
||||
let plan = plan_for(outcome, local);
|
||||
let count = plan.new.len() as u64;
|
||||
let bytes = plan
|
||||
.new
|
||||
.iter()
|
||||
.filter_map(|id| outcome.sizes.get(&id.id).copied())
|
||||
.sum();
|
||||
(count, bytes)
|
||||
}
|
||||
|
||||
fn diff_delta(
|
||||
server_changes: &[EnumeratedItem],
|
||||
deletions: &[String],
|
||||
|
||||
@@ -534,6 +534,74 @@ fn coordinator_diff_classifies_new_vanished_changed_unchanged() {
|
||||
assert_eq!(plan.vanished[0].0, "Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_counts_new_items_and_sums_their_sizes() {
|
||||
use inbuxa_migrate::db::exchange_ews_ids::ItemRow;
|
||||
use inbuxa_migrate::sync::import_exchange_ews::items::{
|
||||
EnumeratedItem, EnumerationMode, EnumerationOutcome, new_count_and_bytes,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
// Three items on the server; A is already local (not new), B and C are new.
|
||||
let items = vec![
|
||||
EnumeratedItem {
|
||||
element: "Message".into(),
|
||||
id: ItemId::new("A", "ck-1"),
|
||||
},
|
||||
EnumeratedItem {
|
||||
element: "Message".into(),
|
||||
id: ItemId::new("B", "ck-1"),
|
||||
},
|
||||
EnumeratedItem {
|
||||
element: "Message".into(),
|
||||
id: ItemId::new("C", "ck-1"),
|
||||
},
|
||||
];
|
||||
let mut sizes = HashMap::new();
|
||||
sizes.insert("A".to_string(), 1000u64);
|
||||
sizes.insert("B".to_string(), 2048u64);
|
||||
// C has no reported size: it still counts, but adds no bytes.
|
||||
let outcome = EnumerationOutcome {
|
||||
items,
|
||||
mode: EnumerationMode::Full,
|
||||
sizes,
|
||||
};
|
||||
let local = vec![ItemRow {
|
||||
item_id: "A".into(),
|
||||
change_key: "ck-1".into(),
|
||||
local_id: 1,
|
||||
}];
|
||||
|
||||
let (count, bytes) = new_count_and_bytes(&outcome, &local);
|
||||
assert_eq!(count, 2, "B and C are new; A is already local");
|
||||
assert_eq!(
|
||||
bytes, 2048,
|
||||
"only B's size is known; C has none; A is not new"
|
||||
);
|
||||
|
||||
// Nothing new once everything is local: no count, no bytes.
|
||||
let all_local = vec![
|
||||
ItemRow {
|
||||
item_id: "A".into(),
|
||||
change_key: "ck-1".into(),
|
||||
local_id: 1,
|
||||
},
|
||||
ItemRow {
|
||||
item_id: "B".into(),
|
||||
change_key: "ck-1".into(),
|
||||
local_id: 2,
|
||||
},
|
||||
ItemRow {
|
||||
item_id: "C".into(),
|
||||
change_key: "ck-1".into(),
|
||||
local_id: 3,
|
||||
},
|
||||
];
|
||||
let (count, bytes) = new_count_and_bytes(&outcome, &all_local);
|
||||
assert_eq!(count, 0);
|
||||
assert_eq!(bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_change_protection_refuses_different_account() {
|
||||
use inbuxa_migrate::db;
|
||||
|
||||
Reference in new issue
Block a user