From 7862aa9bc242a4444333c3a43aff107437b131ac Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 13 Nov 2026 11:11:01 -0700 Subject: [PATCH] Add the reconcile command: a per-folder report that a migration was complete reconcile reads a local archive and reports it folder by folder -- a message count and a content checksum for each folder, totals and a checksum for the whole, and an integrity check that every message resolves to stored content and a real folder. It is read-only and reaches no network, like inspect. Two uses: compare the per-folder counts against the source mailbox to prove the move brought everything across, and compare two archives' checksums to prove they hold the very same mail (an export and a re-import, or a backup and its source). It exits 8 when the archive is not sound, past the 1-7 the other commands use, so a script can tell a reconciliation failure from any other error. --- CHANGELOG.md | 7 + README.md | 8 +- docs/usage.md | 21 ++- src/cli.rs | 29 ++++ src/lib.rs | 1 + src/main.rs | 14 +- src/reconcile.rs | 429 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 506 insertions(+), 3 deletions(-) create mode 100644 src/reconcile.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0527f52..05c906f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a only an API token; with no `--account-*` it imports the account Fastmail marks primary. Mail always; calendars and contacts where that account advertises them. +- `reconcile ` reads an archive folder by folder and reports it: a + message count and a content checksum for each folder, totals and a checksum + for the whole, and an integrity check that every message resolves to stored + content and a real folder. It is read-only and touches no network. Compare + the per-folder counts against the source mailbox to prove a migration was + complete; the checksum shows two archives -- an export and a re-import, say -- + hold the very same mail. It exits 8 when the archive is not sound. ## [27.1-rc.13] diff --git a/README.md b/README.md index 14c7d16..91fcf94 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate import imap … alice.sqlite an account into an archive inbuxa-migrate inspect alice.sqlite what landed + inbuxa-migrate reconcile alice.sqlite folder by folder, is it all there inbuxa-migrate export … alice.sqlite the archive into inbuxa inbuxa-migrate export-files alice.sqlite out the archive as plain files inbuxa-migrate import files out alice.sqlite those files read back, manifest and all @@ -43,7 +44,12 @@ nothing. It reports sizes for `imap`, `maildir`, `caldav`, `carddav`, Exchange contacts that are new). For `jmap` and Microsoft Graph it reports counts only: the enumeration gives the new-object count for free, but a size there would cost a body fetch per object. The remaining sources fall back to a plain -dry-run. `inspect` reads an archive and changes nothing. +dry-run. `inspect` reads an archive and changes nothing, and `reconcile` +reads it folder by folder: a count and a content checksum for each, a checksum +for the whole, and a check that every message resolves to stored content and a +real folder. Compare the per-folder counts against the old mailbox to prove the +move was complete, and the checksum shows two archives hold the very same mail. +It exits non-zero if a message is missing its content or its folder. The archive is also a backup. Import on a schedule, keep the file, and restore it later with `export`. An archive remembers which account filled it diff --git a/docs/usage.md b/docs/usage.md index fe84273..12a0a65 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,7 +13,7 @@ and how to install it. ## Commands and global flags ``` -inbuxa-migrate [args...] +inbuxa-migrate [args...] ``` Every command takes a few global flags -- verbosity, worker pool size, retry @@ -897,6 +897,25 @@ Read-only dump of a local archive. This command never opens a network connection - `mailbox` and `filenode` render as a tree (`--limit`/`--offset` are ignored); all other types use a paginated list and respect `--limit` and `--offset`. +## Reconcile + +``` +inbuxa-migrate reconcile +``` + +Read-only report that an archive holds what it should, folder by folder. Like `inspect`, it never opens a network connection and never writes to the archive; while another run holds the archive it says the archive is in use instead of reading it. + +It prints, in order: + +- **Folders** -- one row per mailbox, by full path, with its message count and a content checksum. A message filed in more than one folder is counted in each. The checksum is over the content of the messages in that folder, so two folders -- or two archives -- with the same mail share a checksum, regardless of order. +- **Totals** -- the message count, the number of folders, the stored content size and blob count, and a single checksum over the whole archive's mail. +- **Integrity** -- either `ok`, or a line for each problem found: messages that point at content not in the archive, empty stored blobs, messages filed in no folder, and messages filed in a folder that is not in the archive. + +Two uses. To prove a migration was complete, compare the per-folder counts against the old mailbox. To prove two archives hold the very same mail -- an export and a re-import, or a backup and its source -- compare their checksums. + +Exit codes: `0` when the archive reconciles cleanly, `8` when an integrity check found a problem (past the `1`--`7` the other commands use, so a script can tell a reconciliation failure from any other error), and `7` when the archive can't be opened or read. + + ## Live tests The tests against real servers -- a Stalwart server as a JMAP source, and diff --git a/src/cli.rs b/src/cli.rs index 994624d..e7257f7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -24,6 +24,7 @@ use crate::jmap::http::Auth; use crate::logging::Logger; use crate::migrate::MigrateConfig; use crate::profile::Profile; +use crate::reconcile::ReconcileConfig; use crate::secret; use crate::sync::export_dav::ExportDavConfig; use crate::sync::export_imap::ExportImapConfig; @@ -108,6 +109,8 @@ enum Command { Plan(Box), #[command(about = "Show the contents of a local archive (read-only)")] Inspect(Box), + #[command(about = "Check a local archive folder by folder and report its contents (read-only)")] + Reconcile(Box), /// Print a shell completion script to standard output /// /// Source it, or install it where the shell looks: @@ -1011,6 +1014,7 @@ pub enum Action { ExportManageSieve(CommonConfig, ExportManageSieveConfig), ExportDav(CommonConfig, ExportDavConfig), Inspect(InspectConfig), + Reconcile(ReconcileConfig), Completions(clap_complete::Shell), Man, } @@ -1209,6 +1213,7 @@ impl Cli { Command::ExportCaldav(args) => resolve_export_dav(*args, DavKindArg::Caldav), Command::ExportCarddav(args) => resolve_export_dav(*args, DavKindArg::Carddav), Command::Inspect(args) => resolve_inspect(*args), + Command::Reconcile(args) => resolve_reconcile(*args), Command::Completions { shell } => Ok(Action::Completions(shell)), Command::Man => Ok(Action::Man), } @@ -1872,6 +1877,18 @@ fn resolve_inspect(args: InspectArgs) -> Result { })) } +#[derive(Args)] +struct ReconcileArgs { + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, +} + +fn resolve_reconcile(args: ReconcileArgs) -> Result { + Ok(Action::Reconcile(ReconcileConfig { + archive: args.archive, + })) +} + #[derive(Args)] #[command(group(clap::ArgGroup::new("ews_auth").required(true).args([ "auth_basic", "auth_bearer", @@ -2512,6 +2529,18 @@ mod tests { ); } + #[test] + fn reconcile_resolves_with_its_archive() { + let cli = Cli::try_parse_from(["inbuxa-migrate", "reconcile", "alice.sqlite"]) + .unwrap_or_else(|e| panic!("clap rejected args: {e}")); + match cli.resolve().unwrap() { + Action::Reconcile(cfg) => assert_eq!(cfg.archive, PathBuf::from("alice.sqlite")), + _ => panic!("expected a reconcile action"), + } + // The archive is required. + assert!(Cli::try_parse_from(["inbuxa-migrate", "reconcile"]).is_err()); + } + #[test] fn graph_max_rps_is_off_by_default_and_parsed_when_given() { assert_eq!(graph_config(&[]).unwrap().max_rps, None); diff --git a/src/lib.rs b/src/lib.rs index bdef6f1..2b4ba0f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod managesieve; pub mod migrate; pub mod net; pub mod profile; +pub mod reconcile; pub mod secret; pub mod sync; pub mod types; diff --git a/src/main.rs b/src/main.rs index 38ca12a..75a1a8d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,9 @@ use clap::{CommandFactory, Parser}; use inbuxa_migrate::cli::{Action, Cli}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::{self, RunOutcome, Summary}; -use inbuxa_migrate::{export_directory, export_files, guide, import_directory, inspect, migrate}; +use inbuxa_migrate::{ + export_directory, export_files, guide, import_directory, inspect, migrate, reconcile, +}; fn main() { let code = run(); @@ -202,6 +204,16 @@ fn run() -> i32 { Err(err) => fail(&err), }; } + Action::Reconcile(config) => { + return match reconcile::run(config) { + // A sound archive exits 0; one with integrity problems exits 8, + // past the 1-7 the other errors use, so a script can tell a + // reconciliation failure from any other. + Ok(true) => 0, + Ok(false) => 8, + Err(err) => fail(&err), + }; + } // Handled before the match, above. Action::Completions(_) | Action::Man => unreachable!(), }; diff --git a/src/reconcile.rs b/src/reconcile.rs new file mode 100644 index 0000000..f204720 --- /dev/null +++ b/src/reconcile.rs @@ -0,0 +1,429 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! The reconciliation report: a read-only manifest of what a local archive +//! holds, folder by folder, with a content checksum per folder and one for the +//! whole, plus a check that the archive is internally sound. It is how you +//! prove a migration brought everything across: compare the per-folder counts +//! against the source mailbox, and the checksum lets you show two archives -- +//! an export and a re-import, say -- hold the very same mail. +//! +//! Everything comes from the archive itself; it opens nothing on the network. + +use std::collections::HashMap; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use rusqlite::{Connection, OptionalExtension}; + +use crate::db::init; +use crate::error::Error; +use crate::inspect::{format_bytes, format_count}; + +pub struct ReconcileConfig { + pub archive: PathBuf, +} + +/// run writes the report and returns whether the archive reconciled cleanly: +/// false when an integrity check found a problem, so the caller can exit +/// non-zero. +pub fn run(config: ReconcileConfig) -> Result { + let conn = init::open_read_only(&config.archive)?; + let stdout = io::stdout(); + let mut out = stdout.lock(); + report(&conn, &config.archive, &mut out) +} + +fn report(conn: &Connection, archive: &Path, out: &mut impl Write) -> Result { + writeln!(out, "Archive: {}", archive.display())?; + if let Some((kind, url, account, name)) = read_source(conn)? { + let name = name.unwrap_or_else(|| "(unnamed)".to_owned()); + writeln!(out, "Source: {kind} {url} (account {account} / {name})")?; + } else { + writeln!(out, "Source: (no source recorded)")?; + } + writeln!(out)?; + + let folders = folder_manifest(conn)?; + let mut total_messages = 0u64; + let mut hasher = blake3::Hasher::new(); + + writeln!(out, "Folders")?; + if folders.is_empty() { + writeln!(out, " (no folders)")?; + } else { + let path_width = folders + .iter() + .map(|f| f.path.len()) + .max() + .unwrap_or(0) + .max("Folder".len()); + let count_width = folders + .iter() + .map(|f| format_count(f.messages).len()) + .max() + .unwrap_or(1) + .max("messages".len()); + writeln!( + out, + " {:cw$} checksum", + "Folder", + "messages", + pw = path_width, + cw = count_width + )?; + for f in &folders { + total_messages += f.messages; + // Fold each folder's checksum into the whole-archive one, so the + // grand total fingerprints the entire mail set, order and all. + hasher.update(f.path.as_bytes()); + hasher.update(&f.checksum); + writeln!( + out, + " {:cw$} {}", + f.path, + format_count(f.messages), + short_hex(&f.checksum), + pw = path_width, + cw = count_width + )?; + } + } + + let (blob_count, blob_bytes) = blob_totals(conn)?; + writeln!(out)?; + writeln!(out, "Totals")?; + writeln!(out, " messages {}", format_count(total_messages))?; + writeln!( + out, + " folders {}", + format_count(folders.len() as u64) + )?; + writeln!( + out, + " stored content {} in {} blobs", + format_bytes(blob_bytes), + format_count(blob_count) + )?; + writeln!( + out, + " checksum {}", + short_hex(hasher.finalize().as_bytes()) + )?; + + let problems = integrity(conn)?; + writeln!(out)?; + writeln!(out, "Integrity")?; + if problems.is_empty() { + writeln!( + out, + " ok -- every message resolves to stored content and a real folder" + )?; + Ok(true) + } else { + for p in &problems { + writeln!(out, " PROBLEM {p}")?; + } + Ok(false) + } +} + +struct Folder { + path: String, + messages: u64, + checksum: [u8; 32], +} + +/// folder_manifest returns one row per mailbox, by full path, with its message +/// count and a checksum over the sorted content hashes of the messages in it. +/// A message in two folders is counted in each, as it is filed in each. +fn folder_manifest(conn: &Connection) -> Result, Error> { + // id -> (parent, name), to build full paths. + let mut names: HashMap, String)> = HashMap::new(); + { + let mut stmt = conn.prepare("SELECT id, parent_id, name FROM mailboxes")?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, Option>(1)?, + r.get::<_, String>(2)?, + )) + })?; + for row in rows { + let (id, parent, name) = row?; + names.insert(id, (parent, name)); + } + } + + let mut folders = Vec::with_capacity(names.len()); + for &id in names.keys() { + let mut hashes = folder_hashes(conn, id)?; + hashes.sort_unstable(); + let mut hasher = blake3::Hasher::new(); + for h in &hashes { + hasher.update(h); + } + folders.push(Folder { + path: full_path(id, &names), + messages: hashes.len() as u64, + checksum: *hasher.finalize().as_bytes(), + }); + } + folders.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(folders) +} + +/// folder_hashes is the raw content hashes of every message filed in a mailbox. +fn folder_hashes(conn: &Connection, mailbox_id: i64) -> Result>, Error> { + let mut stmt = conn.prepare( + "SELECT b.hash FROM emails e JOIN blobs b ON b.id = e.blob_id + WHERE EXISTS (SELECT 1 FROM json_each(e.mailbox_ids) j + WHERE CAST(j.value AS INTEGER) = ?1)", + )?; + let rows = stmt.query_map([mailbox_id], |r| r.get::<_, Vec>(0))?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +fn full_path(id: i64, names: &HashMap, String)>) -> String { + let mut parts = Vec::new(); + let mut cur = Some(id); + // A depth bound guards against a cycle a corrupt archive might hold. + let mut guard = 0; + while let Some(c) = cur { + guard += 1; + if guard > 1000 { + break; + } + match names.get(&c) { + Some((parent, name)) => { + parts.push(name.clone()); + cur = *parent; + } + None => break, + } + } + parts.reverse(); + parts.join("/") +} + +fn blob_totals(conn: &Connection) -> Result<(u64, u64), Error> { + let (count, bytes): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(length(data)), 0) FROM blobs", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok((count.max(0) as u64, bytes.max(0) as u64)) +} + +/// integrity returns a line for each way the archive is unsound, or an empty +/// list when it is whole. These are the failures that make a migration look +/// done when it is not: a message with no content, with no folder, or filed in +/// a folder that is not there. +fn integrity(conn: &Connection) -> Result, Error> { + let mut problems = Vec::new(); + + let dangling: i64 = conn.query_row( + "SELECT COUNT(*) FROM emails e LEFT JOIN blobs b ON b.id = e.blob_id WHERE b.id IS NULL", + [], + |r| r.get(0), + )?; + if dangling > 0 { + problems.push(format!( + "{} message(s) point at content that is not in the archive", + format_count(dangling as u64) + )); + } + + let empty_blobs: i64 = conn.query_row( + "SELECT COUNT(*) FROM blobs WHERE length(data) = 0", + [], + |r| r.get(0), + )?; + if empty_blobs > 0 { + problems.push(format!( + "{} stored blob(s) are empty", + format_count(empty_blobs as u64) + )); + } + + let orphans: i64 = conn.query_row( + "SELECT COUNT(*) FROM emails WHERE json_array_length(mailbox_ids) = 0", + [], + |r| r.get(0), + )?; + if orphans > 0 { + problems.push(format!( + "{} message(s) are filed in no folder", + format_count(orphans as u64) + )); + } + + let bad_refs: i64 = conn.query_row( + "SELECT COUNT(*) FROM emails e + WHERE EXISTS (SELECT 1 FROM json_each(e.mailbox_ids) j + WHERE NOT EXISTS (SELECT 1 FROM mailboxes m + WHERE m.id = CAST(j.value AS INTEGER)))", + [], + |r| r.get(0), + )?; + if bad_refs > 0 { + problems.push(format!( + "{} message(s) are filed in a folder that is not in the archive", + format_count(bad_refs as u64) + )); + } + + Ok(problems) +} + +type SourceRow = (String, String, String, Option); + +fn read_source(conn: &Connection) -> Result, Error> { + let row = conn + .query_row( + "SELECT kind, session_url, account_id, account_name FROM sources LIMIT 1", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .optional()?; + Ok(row) +} + +fn short_hex(hash: &[u8]) -> String { + let mut s = String::with_capacity(16); + for b in hash.iter().take(8) { + s.push_str(&format!("{b:02x}")); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::blobs; + use rusqlite::params; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + fn mailbox(c: &Connection, name: &str, parent: Option) -> i64 { + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, NULL, 0, 1)", + params![name, parent], + ) + .unwrap(); + c.last_insert_rowid() + } + + fn email(c: &Connection, body: &[u8], mailbox_ids: &[i64]) { + let blob = blobs::intern_blob(c, body).unwrap(); + let ids: Vec = mailbox_ids.iter().map(|i| i.to_string()).collect(); + c.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-01-01T00:00:00Z', ?2, '[]')", + params![blob, format!("[{}]", ids.join(","))], + ) + .unwrap(); + } + + fn render(c: &Connection) -> (String, bool) { + let mut buf = Vec::new(); + let ok = report(c, Path::new("test.sqlite"), &mut buf).unwrap(); + (String::from_utf8(buf).unwrap(), ok) + } + + #[test] + fn counts_messages_per_folder_and_reconciles_cleanly() { + let c = mem(); + let inbox = mailbox(&c, "INBOX", None); + let archived = mailbox(&c, "Archive", None); + email(&c, b"one", &[inbox]); + email(&c, b"two", &[inbox]); + email(&c, b"three", &[archived]); + let (out, ok) = render(&c); + assert!(ok, "a sound archive reconciles: {out}"); + assert!(out.contains("INBOX"), "{out}"); + assert!(out.contains("Archive"), "{out}"); + // INBOX has two, Archive one. + let inbox_line = out.lines().find(|l| l.contains("INBOX")).unwrap(); + assert!(inbox_line.contains('2'), "INBOX line: {inbox_line}"); + } + + #[test] + fn nested_folders_show_their_full_path() { + let c = mem(); + let parent = mailbox(&c, "Archive", None); + let child = mailbox(&c, "2019", Some(parent)); + email(&c, b"old", &[child]); + let (out, ok) = render(&c); + assert!(ok); + assert!(out.contains("Archive/2019"), "want full path: {out}"); + } + + #[test] + fn the_same_message_in_two_folders_is_counted_in_each() { + let c = mem(); + let a = mailbox(&c, "A", None); + let b = mailbox(&c, "B", None); + email(&c, b"shared", &[a, b]); + let (out, _) = render(&c); + // Two folder rows, each with one message, from a single stored blob. + assert!( + out.contains("1 blobs") || out.contains("in 1 blobs"), + "{out}" + ); + } + + #[test] + fn identical_content_gives_two_folders_the_same_checksum() { + let c = mem(); + let a = mailbox(&c, "A", None); + let b = mailbox(&c, "B", None); + email(&c, b"identical", &[a]); + email(&c, b"identical", &[b]); + let folders = folder_manifest(&c).unwrap(); + let ca = &folders.iter().find(|f| f.path == "A").unwrap().checksum; + let cb = &folders.iter().find(|f| f.path == "B").unwrap().checksum; + assert_eq!(ca, cb, "same content => same folder checksum"); + } + + #[test] + fn a_message_filed_in_a_missing_folder_is_a_problem() { + let c = mem(); + let real = mailbox(&c, "INBOX", None); + email(&c, b"ok", &[real]); + email(&c, b"lost", &[9999]); // 9999 is not a mailbox + let (out, ok) = render(&c); + assert!(!ok, "a dangling folder reference must not reconcile: {out}"); + assert!(out.contains("not in the archive"), "{out}"); + } + + #[test] + fn a_message_in_no_folder_is_a_problem() { + let c = mem(); + mailbox(&c, "INBOX", None); + email(&c, b"homeless", &[]); + let (out, ok) = render(&c); + assert!(!ok); + assert!(out.contains("filed in no folder"), "{out}"); + } + + #[test] + fn an_empty_archive_reports_no_folders_and_reconciles() { + let c = mem(); + let (out, ok) = render(&c); + assert!(ok); + assert!(out.contains("(no folders)"), "{out}"); + } +} -- 2.54.0