estimate: counts-only for JMAP and Microsoft Graph #72
No files matched your search
@@ -40,8 +40,10 @@ takes `--dry-run` and then only reports the plan. An `import` also takes
|
||||
fetch — counts, bytes and a rough time — before committing, and writes
|
||||
nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange
|
||||
(`ews`) and a `takeout` (whose files it reads to count the mail, events and
|
||||
contacts that are new); other sources fall back to a plain dry-run. `inspect`
|
||||
reads an archive and changes nothing.
|
||||
contacts that are new). For `jmap` and Microsoft Graph it reports counts only:
|
||||
the enumeration gives the new-object count for free, but a size there would
|
||||
cost a body fetch per object. The remaining sources fall back to a plain
|
||||
dry-run. `inspect` reads an archive and changes nothing.
|
||||
|
||||
The archive is also a backup. Import on a schedule, keep the file, and
|
||||
restore it later with `export`. An archive remembers which account filled it
|
||||
|
||||
+8
-3
@@ -156,9 +156,10 @@ struct GlobalArgs {
|
||||
help = "Preview an import: counts, sizes and a rough time, before any writes",
|
||||
long_help = "Preview an import before committing: for each type, how much is new since \
|
||||
the archive was last filled, how many bytes that is, and a rough time.\n \
|
||||
Available for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, \
|
||||
events and contacts) today; other sources fall back to --dry-run with a note. It \
|
||||
implies --dry-run, so it never writes. On an export it is just --dry-run."
|
||||
Sizes for imap, maildir, caldav, carddav, exchange (EWS) and takeout (mail, events \
|
||||
and contacts); counts only for jmap and Graph (a size there needs a body fetch per \
|
||||
object). Other sources fall back to --dry-run with a note. It implies --dry-run, so \
|
||||
it never writes. On an export it is just --dry-run."
|
||||
)]
|
||||
estimate: bool,
|
||||
|
||||
@@ -936,6 +937,10 @@ impl Action {
|
||||
// EWS FindItem returns item:Size, so mail, calendars and contacts
|
||||
// are all counted and sized from the enumeration.
|
||||
Action::ImportExchangeEws(c, _) => c.estimate,
|
||||
// JMAP and Graph report counts only: the dry-run enumeration gives
|
||||
// the new-object count for free, but a size would cost a body fetch
|
||||
// per object, which an estimate does not do.
|
||||
Action::Import(c, _) | Action::ImportExchangeGraph(c, _) => c.estimate,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ fn run() -> i32 {
|
||||
if action.estimate_requested() && !do_estimate {
|
||||
eprintln!(
|
||||
"note: --estimate isn't available for this source yet; running --dry-run instead \
|
||||
(its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS) and takeout today."
|
||||
(its plan is below). Estimates work for imap, maildir, caldav, carddav, exchange (EWS), takeout, jmap and Graph today."
|
||||
);
|
||||
}
|
||||
let mut quiet_report = false;
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
pub mod calendar;
|
||||
pub mod contacts;
|
||||
pub mod coordinator;
|
||||
pub mod estimate;
|
||||
pub mod files;
|
||||
pub mod folders;
|
||||
pub mod messages;
|
||||
|
||||
@@ -114,6 +114,30 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
|
||||
}
|
||||
|
||||
if common.dry_run {
|
||||
if common.estimate {
|
||||
let source_id = db::sources::find_source(
|
||||
&conn,
|
||||
&SourceKey {
|
||||
kind: "exchange_graph".to_owned(),
|
||||
session_url: session_url.clone(),
|
||||
account_id: account_id.clone(),
|
||||
},
|
||||
)?
|
||||
.unwrap_or(-1);
|
||||
let ctx = GraphCoordinator {
|
||||
client: &client,
|
||||
endpoints: &endpoints,
|
||||
source_id,
|
||||
top: config.top.clamp(1, 1000),
|
||||
workers: config.graph_connections.clamp(1, 16),
|
||||
logger,
|
||||
event_body_format: config.event_body_format,
|
||||
exception_window_years: config.exception_window_years,
|
||||
contact_photos: config.contact_photos,
|
||||
event_attachments: config.event_attachments,
|
||||
};
|
||||
return super::estimate::run(&conn, &ctx, config.mailbox_kind, &config.surfaces);
|
||||
}
|
||||
return run_dry(
|
||||
&client,
|
||||
&endpoints,
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! The counts-only `--estimate` pass for a Microsoft Graph source.
|
||||
//!
|
||||
//! Graph can enumerate an item's id cheaply (`$select=id`), but a size would
|
||||
//! cost a body fetch per item, which an estimate does not do. So this reports
|
||||
//! how many mail, calendar events and contacts are new since the archive was
|
||||
//! last filled -- the same id enumeration and archive diff the import runs --
|
||||
//! and no bytes. It reads folder lists and id pages only; it writes nothing.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use rusqlite::Connection;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::coordinator::{GraphCoordinator, enumerate_mail_folders};
|
||||
use super::folders::default_contact_folder;
|
||||
use crate::db::exchange_graph_ids;
|
||||
use crate::error::Error;
|
||||
use crate::exchange_graph::api::{self, PREFER_TIMEZONE_UTC};
|
||||
use crate::exchange_graph::calendar_map::{EventType, classify_event_type};
|
||||
use crate::exchange_graph::types::{MailboxKind, Surfaces};
|
||||
use crate::sync::{Summary, TypeCounts};
|
||||
|
||||
/// Builds the estimate summary: a per-type row of new objects for each surface
|
||||
/// the run asks for. Mirrors which surfaces the real import would touch (an
|
||||
/// archive mailbox carries mail only).
|
||||
pub fn run(
|
||||
conn: &Connection,
|
||||
ctx: &GraphCoordinator<'_>,
|
||||
mailbox_kind: MailboxKind,
|
||||
surfaces: &Surfaces,
|
||||
) -> Result<Summary, Error> {
|
||||
let mut summary = Summary::default();
|
||||
let primary = !matches!(mailbox_kind, MailboxKind::Archive);
|
||||
if surfaces.mail {
|
||||
summary
|
||||
.per_type
|
||||
.push(("email", estimate_mail(conn, ctx, mailbox_kind)?));
|
||||
}
|
||||
if surfaces.calendar && primary {
|
||||
summary
|
||||
.per_type
|
||||
.push(("calendarevent", estimate_calendar(conn, ctx)?));
|
||||
}
|
||||
if surfaces.contacts && primary {
|
||||
summary
|
||||
.per_type
|
||||
.push(("contactcard", estimate_contacts(conn, ctx)?));
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Counts the mail that is new across the full folder tree. Enumerates message
|
||||
/// ids folder by folder -- the same read-only tree walk a dry run does -- and
|
||||
/// counts those not already in the archive, deduplicated across folders.
|
||||
fn estimate_mail(
|
||||
conn: &Connection,
|
||||
ctx: &GraphCoordinator<'_>,
|
||||
mailbox_kind: MailboxKind,
|
||||
) -> Result<TypeCounts, Error> {
|
||||
let local = exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?;
|
||||
let folders = enumerate_mail_folders(ctx.client, ctx.endpoints, mailbox_kind, ctx.top)
|
||||
.map_err(Error::from)?;
|
||||
let mut counts = TypeCounts::default();
|
||||
let mut planned: HashSet<String> = HashSet::new();
|
||||
for folder in &folders {
|
||||
let Some(fid) = folder.get("id").and_then(Value::as_str) else {
|
||||
continue;
|
||||
};
|
||||
let url = ctx.endpoints.folder_messages_ids(fid, ctx.top);
|
||||
let stubs = match api::collect_all_values(ctx.client, &url, &[]) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
ctx.logger
|
||||
.warn(&format!("estimate: mail folder {fid} enumerate: {e}"));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for stub in &stubs {
|
||||
count_new(
|
||||
stub.get("id").and_then(Value::as_str),
|
||||
&local,
|
||||
&mut planned,
|
||||
&mut counts,
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(counts)
|
||||
}
|
||||
|
||||
/// Counts the calendar events that are new. Enumerates event ids per calendar
|
||||
/// and skips single `Occurrence` items exactly as the import does -- it stores
|
||||
/// the series master, not each occurrence -- so a re-estimate finds nothing
|
||||
/// new once imported.
|
||||
fn estimate_calendar(conn: &Connection, ctx: &GraphCoordinator<'_>) -> Result<TypeCounts, Error> {
|
||||
let local =
|
||||
exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?;
|
||||
let calendars = api::collect_all_values(ctx.client, &ctx.endpoints.calendars(ctx.top), &[])
|
||||
.map_err(Error::from)?;
|
||||
let mut counts = TypeCounts::default();
|
||||
let mut planned: HashSet<String> = HashSet::new();
|
||||
for cal in &calendars {
|
||||
let Some(cid) = cal.get("id").and_then(Value::as_str) else {
|
||||
continue;
|
||||
};
|
||||
let url = ctx.endpoints.calendar_events_ids(cid, ctx.top);
|
||||
let stubs = match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
ctx.logger
|
||||
.warn(&format!("estimate: calendar {cid} enumerate: {e}"));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for stub in &stubs {
|
||||
if matches!(classify_event_type(stub), EventType::Occurrence) {
|
||||
continue;
|
||||
}
|
||||
count_new(
|
||||
stub.get("id").and_then(Value::as_str),
|
||||
&local,
|
||||
&mut planned,
|
||||
&mut counts,
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(counts)
|
||||
}
|
||||
|
||||
/// Counts the contacts that are new across every address book, including nested
|
||||
/// folders and the default folder, which is the same set the import walks.
|
||||
fn estimate_contacts(conn: &Connection, ctx: &GraphCoordinator<'_>) -> Result<TypeCounts, Error> {
|
||||
let local =
|
||||
exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?;
|
||||
let books = discover_contact_folder_ids(ctx)?;
|
||||
let mut counts = TypeCounts::default();
|
||||
let mut planned: HashSet<String> = HashSet::new();
|
||||
for bid in &books {
|
||||
let url = ctx.endpoints.contact_folder_contacts_ids(bid, ctx.top);
|
||||
let ids = match api::collect_all_ids(ctx.client, &url, &[]) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
ctx.logger
|
||||
.warn(&format!("estimate: contact folder {bid} enumerate: {e}"));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for id in &ids {
|
||||
count_new(Some(id.as_str()), &local, &mut planned, &mut counts);
|
||||
}
|
||||
}
|
||||
Ok(counts)
|
||||
}
|
||||
|
||||
/// The read-only address-book discovery the import does: the listed folders,
|
||||
/// the default folder (which may not be listed), and every nested child.
|
||||
fn discover_contact_folder_ids(ctx: &GraphCoordinator<'_>) -> Result<Vec<String>, Error> {
|
||||
let server = api::collect_all_values(ctx.client, &ctx.endpoints.contact_folders(ctx.top), &[])
|
||||
.map_err(Error::from)?;
|
||||
let mut seen: HashSet<String> = server
|
||||
.iter()
|
||||
.filter_map(|f| f.get("id").and_then(Value::as_str))
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
if let Some(default) = default_contact_folder(ctx)
|
||||
&& let Some(id) = default.get("id").and_then(Value::as_str)
|
||||
{
|
||||
seen.insert(id.to_owned());
|
||||
}
|
||||
let mut frontier: Vec<String> = seen.iter().cloned().collect();
|
||||
while let Some(parent) = frontier.pop() {
|
||||
let url = ctx.endpoints.contact_folder_children(&parent, ctx.top);
|
||||
let children = api::collect_all_values(ctx.client, &url, &[]).map_err(Error::from)?;
|
||||
for child in children {
|
||||
if let Some(id) = child.get("id").and_then(Value::as_str)
|
||||
&& seen.insert(id.to_owned())
|
||||
{
|
||||
frontier.push(id.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(seen.into_iter().collect())
|
||||
}
|
||||
|
||||
/// Counts an enumerated id as new when it is neither already in the archive nor
|
||||
/// already seen in this pass (the same id can appear in two folders).
|
||||
fn count_new(
|
||||
id: Option<&str>,
|
||||
local: &HashMap<String, i64>,
|
||||
planned: &mut HashSet<String>,
|
||||
counts: &mut TypeCounts,
|
||||
) {
|
||||
let Some(id) = id else { return };
|
||||
if local.contains_key(id) {
|
||||
return;
|
||||
}
|
||||
if planned.insert(id.to_owned()) {
|
||||
counts.created += 1;
|
||||
counts.fetched += 1;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn count_new_skips_archived_and_dedups_within_a_pass() {
|
||||
let mut local: HashMap<String, i64> = HashMap::new();
|
||||
local.insert("archived".to_owned(), 1);
|
||||
let mut planned: HashSet<String> = HashSet::new();
|
||||
let mut counts = TypeCounts::default();
|
||||
|
||||
// Already in the archive: not new.
|
||||
count_new(Some("archived"), &local, &mut planned, &mut counts);
|
||||
// New: counted once...
|
||||
count_new(Some("A"), &local, &mut planned, &mut counts);
|
||||
// ...and the same id in another folder does not count again.
|
||||
count_new(Some("A"), &local, &mut planned, &mut counts);
|
||||
// Another new one.
|
||||
count_new(Some("B"), &local, &mut planned, &mut counts);
|
||||
// A stub with no id is ignored.
|
||||
count_new(None, &local, &mut planned, &mut counts);
|
||||
|
||||
assert_eq!(counts.fetched, 2, "A and B are new; A counts once");
|
||||
assert_eq!(counts.created, 2);
|
||||
assert_eq!(counts.bytes, 0, "counts-only: no sizes");
|
||||
}
|
||||
}
|
||||
@@ -341,7 +341,7 @@ pub fn reconcile_address_books(
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option<Value> {
|
||||
pub(crate) fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option<Value> {
|
||||
match ctx.client.get_json(&ctx.endpoints.default_contact_folder()) {
|
||||
Ok(value) if value.get("id").and_then(Value::as_str).is_some() => return Some(value),
|
||||
Ok(_) => {}
|
||||
|
||||
+10
-2
@@ -298,8 +298,12 @@ fn run_into(
|
||||
}
|
||||
|
||||
if ctx.dry_run() {
|
||||
print_dry_run(&dry_rows);
|
||||
*summary = Summary::default();
|
||||
// A plain dry run prints its own table and discards the counts; an
|
||||
// estimate keeps the per-type summary so the caller can render it.
|
||||
if !ctx.estimate() {
|
||||
print_dry_run(&dry_rows);
|
||||
*summary = Summary::default();
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -399,6 +403,10 @@ fn reconcile_type(
|
||||
d.vanished.len() as u64,
|
||||
d.present.len() as u64,
|
||||
));
|
||||
// --estimate keeps the summary (counts only; JMAP would need a body
|
||||
// fetch per object to size it, which an estimate does not do), so the
|
||||
// new-object count has to land in the per-type row the caller pushes.
|
||||
counts.fetched = d.new.len() as u64;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user