From 687027c7cbe1a044ab88799666b6260f8df10223 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 30 Sep 2026 11:32:04 -0700 Subject: [PATCH] export: bring matched items up to date on every run Export matched each item against the target and then skipped it, so a second run -- the usual final pass of a cutover -- never carried anything that had changed at the source since the first: read and flagged state, moves between folders, edited contacts, events and Sieve scripts. It reported them as skipped and exited 0, while the usage guide said matched items were updated. Matched items are now updated, with one batched /set per type: - Email: keywords are set to the archive's, added and removed, compared case-insensitively. Memberships of folders this run migrated are added and removed to match; folders that exist only on the target are left alone, and a message is never left in no folder. Properties the server did not report are not touched. - Contacts and events: when both copies carry `updated`, the archive's is written only if it is newer, compared as instants so an offset or a fraction of a second is not taken for a change; otherwise each property the archive writes is compared, and those that differ are sent whole. - Sieve scripts: the target's copy is downloaded and compared byte for byte with what export would write -- after renaming Stalwart's vendor names for an inbuxa target -- and replaced when it differs, so a renamed script is not re-uploaded on every run. Updated items are counted as `updated`; unchanged ones stay `skipped`. The usage guide now describes this. --- docs/usage.md | 25 ++- src/sync/export.rs | 60 +++++- src/sync/export/email.rs | 208 +++++++++++++------ src/sync/export/sieve.rs | 107 +++++++--- src/sync/export/uidtype.rs | 105 +++++++++- tests/mock_sync.rs | 410 ++++++++++++++++++++++++++++++++++++- 6 files changed, 803 insertions(+), 112 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 5e25826..4802889 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -211,16 +211,23 @@ inbuxa-migrate export \ Writes `ARCHIVE` into an account on a JMAP server, usually inbuxa. It keeps no state of its own: every run matches the archive against the target afresh. -By default it only adds: what the archive holds and the target lacks is -created, and anything already on the target is left as it is -- an item that -matches is not updated yet, so a change made at the source after the first -export does not reach the target on a second one. +By default it only adds and updates: what the target lacks is created, what +it has is brought up to date, and anything on the target that the archive +does not cover is left alone. So a second run after a later import carries +what changed at the source in between. -Email is matched by Message-ID, or without one by sender, subject, date and -recipients; where several messages share one, size decides. A message the -source kept in several folders -- IMAP and Maildir copies, Gmail labels -- is -written once, in all of them, and a later run adds any folder it is still -missing on the target. +- **Email** is matched by Message-ID, or without one by sender, subject, date + and recipients; where several messages share one, size decides. A message + the source kept in several folders -- IMAP and Maildir copies, Gmail labels + -- is written once, in all of them. On a match, its keywords (read, + flagged and the rest) are set to the archive's, and so are its memberships + of folders this run migrated. Folders that exist only on the target are + left alone, and a message is never left in no folder. +- **Contacts and events** are matched by UID. When both copies carry an + `updated` time, the archive's is written only if it is newer; otherwise + the properties that differ are written. +- **Sieve scripts** are matched by name, and the target's is replaced when + its content differs from the archive's. `--prune` also deletes what is on the target and not in the archive. It asks first; `--yes` answers for it, for scripts. Export speaks JMAP only. diff --git a/src/sync/export.rs b/src/sync/export.rs index d3cadfe..143a2ec 100644 --- a/src/sync/export.rs +++ b/src/sync/export.rs @@ -5,7 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::{IsTerminal, Write}; use rusqlite::Connection; @@ -49,6 +49,14 @@ impl Maps { fn insert(&mut self, ty: ObjectType, local: i64, target: JmapId) { self.m.entry(ty).or_default().insert(local, target); } + + /// Every target id this run mapped for `ty`: the objects it migrated. + fn targets_of(&self, ty: ObjectType) -> HashSet { + self.m + .get(&ty) + .map(|m| m.values().map(|id| id.0.clone()).collect()) + .unwrap_or_default() + } } impl TargetResolver for Maps { @@ -533,6 +541,56 @@ mod common { ) } + /// Sends `updates` (target id, patch) as batched `/set` calls and counts + /// the result into `counts`. A dry run counts them as updated and sends + /// nothing. + pub fn update_batch( + net: &Net, + ty: ObjectType, + updates: Vec<(String, Value)>, + counts: &mut TypeCounts, + logger: &Logger, + ) { + if updates.is_empty() { + return; + } + if net.dry_run { + counts.updated += updates.len() as u64; + return; + } + let total = updates.len() as u64; + let mut map = Map::new(); + for (id, patch) in updates { + map.insert(id, patch); + } + match set_call( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + SetRequest { + update: Some(Value::Object(map)), + ..Default::default() + }, + &net.limits, + ) { + Ok(outcome) => { + counts.updated += outcome.updated.len() as u64; + for (id, err) in &outcome.not_updated { + logger.warn(&format!("{}/set {id} not updated: {err}", ty.jmap_name())); + counts.failed += 1; + } + } + Err(e) => { + logger.warn(&format!( + "{}/set: updating {total} object(s) failed: {e}", + ty.jmap_name() + )); + counts.failed += total; + } + } + } + fn blob_not_found(outcome: &crate::jmap::request::SetOutcome, cid: &str) -> bool { outcome.not_created.iter().any(|(c, err)| { c == cid && err.get("type").and_then(Value::as_str) == Some("blobNotFound") diff --git a/src/sync/export/email.rs b/src/sync/export/email.rs index 9b23e59..b58cb4a 100644 --- a/src/sync/export/email.rs +++ b/src/sync/export/email.rs @@ -9,12 +9,12 @@ use std::collections::{HashMap, HashSet}; use serde_json::{Map, Value, json}; -use super::common::{jid, target_query_get}; +use super::common::{jid, target_query_get, update_batch}; use super::{Maps, Net, Plan, Uploader}; use crate::error::Error; use crate::jmap::error::JmapError; use crate::jmap::request::{ - MethodCall, Request, SetRequest, check_method_error, get_objects, retry_method_call, set_call, + MethodCall, Request, check_method_error, get_objects, retry_method_call, }; use crate::jmap::retry::MethodCallKind; use crate::jmap::wire::JmapId; @@ -62,8 +62,12 @@ pub fn reconcile( ) -> Result { let ty = ObjectType::Email; - let target_min = target_query_get(net, ty, Some(&["messageId", "size", "mailboxIds"])) - .map_err(Error::from)?; + let target_min = target_query_get( + net, + ty, + Some(&["messageId", "size", "mailboxIds", "keywords"]), + ) + .map_err(Error::from)?; let mut indices: Vec = target_min.iter().map(server_index).collect(); let fallback_ids: Vec = target_min @@ -128,11 +132,12 @@ pub fn reconcile( .collect(); let pairs = pair_with_targets(&local_keys, &sizes, &target_keys, &targets); - let mut membership_updates: Vec<(String, Value)> = Vec::new(); + let migrated = maps.targets_of(ObjectType::Mailbox); + let mut updates: Vec<(String, Value)> = Vec::new(); for (i, unit) in units.iter().enumerate() { match pairs[i] { - Some(t) => match missing_memberships(&unit.row, &targets[t], maps) { - Some(patch) => membership_updates.push((targets[t].id.clone(), patch)), + Some(t) => match email_patch(&unit.row, &targets[t], maps, &migrated) { + Some(patch) => updates.push((targets[t].id.clone(), patch)), None => counts.skipped += 1, }, None => export_one( @@ -146,7 +151,7 @@ pub fn reconcile( ), } } - send_membership_updates(net, membership_updates, counts, logger); + update_batch(net, ty, updates, counts, logger); Ok(Plan::default()) } @@ -197,6 +202,7 @@ struct TargetEmail { id: String, size: Option, mailboxes: Option>, + keywords: Option>, } impl TargetEmail { @@ -208,6 +214,10 @@ impl TargetEmail { .get("mailboxIds") .and_then(Value::as_object) .map(|m| m.keys().cloned().collect()), + keywords: v + .get("keywords") + .and_then(Value::as_object) + .map(|m| m.keys().map(|k| k.to_lowercase()).collect()), } } } @@ -254,65 +264,58 @@ fn pair_with_targets( out } -/// The `Email/set` patch adding the folders a matched email is missing on the -/// target, or `None` when it is already in all of them. Folders that exist -/// only on the target are left alone. -fn missing_memberships(row: &EmailRow, target: &TargetEmail, maps: &Maps) -> Option { - let have = target.mailboxes.as_ref()?; +/// The `Email/set` patch that brings a matched email on the target in line +/// with the archive, or `None` when it already is. The source is taken as +/// the truth for what it covers: keywords are added and removed to match, and +/// so are memberships of folders this run migrated. Folders that exist only +/// on the target are left alone, an email is never left in no folder, and +/// whatever the server did not report is not touched. +fn email_patch( + row: &EmailRow, + target: &TargetEmail, + maps: &Maps, + migrated: &HashSet, +) -> Option { let mut patch = Map::new(); - for ml in &row.mailbox_locals { - if let Some(t) = maps.target(ObjectType::Mailbox, *ml) - && !have.contains(&t.0) - { - patch.insert(format!("mailboxIds/{}", t.0), Value::Bool(true)); + if let Some(have) = &target.mailboxes { + let want: HashSet = row + .mailbox_locals + .iter() + .filter_map(|ml| maps.target(ObjectType::Mailbox, *ml).map(|t| t.0)) + .collect(); + let add: Vec<&String> = want.iter().filter(|t| !have.contains(*t)).collect(); + let remove: Vec<&String> = have + .iter() + .filter(|t| migrated.contains(*t) && !want.contains(*t)) + .collect(); + let left = have.len() - remove.len() + add.len(); + for t in add { + patch.insert( + format!("mailboxIds/{}", pointer_escape(t)), + Value::Bool(true), + ); + } + if left > 0 { + for t in remove { + patch.insert(format!("mailboxIds/{}", pointer_escape(t)), Value::Null); + } + } + } + if let Some(have) = &target.keywords { + let want: HashSet = row.keywords.iter().map(|k| k.to_lowercase()).collect(); + for k in want.difference(have) { + patch.insert(format!("keywords/{}", pointer_escape(k)), Value::Bool(true)); + } + for k in have.difference(&want) { + patch.insert(format!("keywords/{}", pointer_escape(k)), Value::Null); } } (!patch.is_empty()).then_some(Value::Object(patch)) } -fn send_membership_updates( - net: &Net, - updates: Vec<(String, Value)>, - counts: &mut TypeCounts, - logger: &Logger, -) { - if updates.is_empty() { - return; - } - if net.dry_run { - counts.updated += updates.len() as u64; - return; - } - let total = updates.len() as u64; - let mut map = Map::new(); - for (id, patch) in updates { - map.insert(id, patch); - } - match set_call( - &net.client, - &net.api, - &net.account, - ObjectType::Email.jmap_name(), - SetRequest { - update: Some(Value::Object(map)), - ..Default::default() - }, - &net.limits, - ) { - Ok(outcome) => { - counts.updated += outcome.updated.len() as u64; - for (id, err) in &outcome.not_updated { - logger.warn(&format!("Email/set {id}: folders not added: {err}")); - counts.failed += 1; - } - } - Err(e) => { - logger.warn(&format!( - "Email/set: adding folders to {total} email(s) failed: {e}" - )); - counts.failed += total; - } - } +/// Escapes one JSON Pointer segment (RFC 6901), as JMAP patch paths use. +fn pointer_escape(segment: &str) -> String { + segment.replace('~', "~0").replace('/', "~1") } fn build_mailbox_ids(row: &EmailRow, maps: &Maps) -> Option> { @@ -570,9 +573,14 @@ mod tests { id: id.to_owned(), size, mailboxes: mailboxes.map(|m| m.iter().map(|s| (*s).to_owned()).collect()), + keywords: None, } } + fn set(items: &[&str]) -> HashSet { + items.iter().map(|s| (*s).to_owned()).collect() + } + fn mid(m: &str) -> EmailKey { EmailKey::MessageId(m.to_owned()) } @@ -628,19 +636,85 @@ mod tests { assert_eq!(pairs, vec![Some(0), None]); } - #[test] - fn missing_memberships_adds_only_the_absent_migrated_folders() { + fn two_folder_maps() -> Maps { let mut maps = Maps::default(); maps.insert(ObjectType::Mailbox, 1, JmapId("T1".into())); maps.insert(ObjectType::Mailbox, 2, JmapId("T2".into())); + maps + } + + #[test] + fn patch_adds_the_absent_migrated_folders() { + let maps = two_folder_maps(); let r = row(10, &[1, 2, 3], &[]); - let patch = missing_memberships(&r, &target("E", None, Some(&["T1", "Own"])), &maps) - .expect("T2 is missing"); + let patch = email_patch( + &r, + &target("E", None, Some(&["T1", "Own"])), + &maps, + &set(&["T1", "T2"]), + ) + .expect("T2 is missing"); assert_eq!(patch, json!({ "mailboxIds/T2": true })); - assert!(missing_memberships(&r, &target("E", None, Some(&["T1", "T2"])), &maps).is_none()); assert!( - missing_memberships(&r, &target("E", None, None), &maps).is_none(), + email_patch( + &r, + &target("E", None, Some(&["T1", "T2"])), + &maps, + &set(&["T1", "T2"]) + ) + .is_none() + ); + assert!( + email_patch(&r, &target("E", None, None), &maps, &set(&["T1", "T2"])).is_none(), "unknown membership is left alone" ); } + + #[test] + fn patch_moves_between_migrated_folders_but_keeps_target_only_ones() { + let maps = two_folder_maps(); + let r = row(10, &[2], &[]); + let patch = email_patch( + &r, + &target("E", None, Some(&["T1", "Own"])), + &maps, + &set(&["T1", "T2"]), + ) + .unwrap(); + assert_eq!( + patch, + json!({ "mailboxIds/T2": true, "mailboxIds/T1": null }) + ); + } + + #[test] + fn patch_never_leaves_an_email_in_no_folder() { + let maps = two_folder_maps(); + let r = row(10, &[9], &[]); + assert!( + email_patch( + &r, + &target("E", None, Some(&["T1"])), + &maps, + &set(&["T1", "T2"]) + ) + .is_none(), + "the only folder is not removed when nothing replaces it" + ); + } + + #[test] + fn patch_syncs_keywords_both_ways_case_insensitively() { + let maps = two_folder_maps(); + let r = row(10, &[1], &["$Seen", "work/urgent"]); + let mut t = target("E", None, Some(&["T1"])); + t.keywords = Some(set(&["$seen", "$flagged"])); + let patch = email_patch(&r, &t, &maps, &set(&["T1", "T2"])).unwrap(); + assert_eq!( + patch, + json!({ "keywords/work~1urgent": true, "keywords/$flagged": null }) + ); + t.keywords = Some(set(&["$seen", "work/urgent"])); + assert!(email_patch(&r, &t, &maps, &set(&["T1", "T2"])).is_none()); + } } diff --git a/src/sync/export/sieve.rs b/src/sync/export/sieve.rs index 5420559..3db9606 100644 --- a/src/sync/export/sieve.rs +++ b/src/sync/export/sieve.rs @@ -9,10 +9,11 @@ use std::collections::{HashMap, HashSet}; use serde_json::{Value, json}; -use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all}; +use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all, update_batch}; use super::sieve_names; use super::{Maps, Net, Plan, Uploader}; use crate::error::Error; +use crate::jmap::blobxfer; use crate::jmap::request::{Request, check_method_error}; use crate::logging::{LEVEL_DEFAULT, Logger}; use crate::sync::import_jmap::mapping::BlobBytes; @@ -31,10 +32,14 @@ pub fn reconcile( let targets = target_get_all(net, ty).map_err(Error::from)?; let mut target_by_name: HashMap = HashMap::new(); + let mut target_blob: HashMap = HashMap::new(); for t in &targets { let (Some(id), Some(name)) = (jid(t), t.get("name").and_then(Value::as_str)) else { continue; }; + if let Some(blob) = t.get("blobId").and_then(Value::as_str) { + target_blob.insert(id.clone(), blob.to_owned()); + } target_by_name.insert(name.to_owned(), id); } @@ -66,19 +71,55 @@ pub fn reconcile( .find(|(_, _, a, _)| *a) .map(|(_, n, _, _)| n.clone().unwrap_or_default()); + let mut updates: Vec<(String, Value)> = Vec::new(); + for (local, name, is_active, blob_local) in &locals { let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned(); + let label = name.as_deref().unwrap_or("(unnamed)"); + let rewritten = if rename_vendor { + renamed_script(&uploader, *blob_local)? + } else { + None + }; let target_id = if let Some(id) = matched { - counts.skipped += 1; + // Compare what would be written -- the renamed bytes where the + // script needed renaming -- so an unchanged script stays unchanged. + let ours = match &rewritten { + Some((bytes, _)) => bytes.clone(), + None => uploader.bytes(*blob_local).map_err(Error::from)?, + }; + match content_differs(net, &ours, target_blob.get(&id)) { + Ok(false) => counts.skipped += 1, + Ok(true) => { + let blob = match &rewritten { + Some((bytes, renamed)) => { + log_renames(label, renamed, logger); + uploader.upload_bytes_as(*blob_local, "application/sieve", bytes) + } + None => uploader.upload_with(*blob_local, "application/sieve"), + }; + match blob { + Ok(b) => updates.push((id.clone(), json!({ "blobId": b.0 }))), + Err(e) => { + logger.warn(&format!( + "SieveScript {label}: upload for update failed: {e}" + )); + counts.failed += 1; + } + } + } + Err(e) => { + logger.warn(&format!("SieveScript {label}: not compared: {e}")); + counts.skipped += 1; + } + } id } else { let cid = format!("c{local}"); - let label = name.as_deref().unwrap_or("(unnamed)"); - let rewritten = if rename_vendor { - renamed_script(&uploader, *blob_local, label, logger)? - } else { - None - }; + if let Some((_, renamed)) = &rewritten { + log_renames(label, renamed, logger); + } + let rewritten = rewritten.as_ref().map(|(bytes, _)| bytes); let build = |up: &mut Uploader<'_>| -> Result { let blob_id = match &rewritten { Some(bytes) => up.upload_bytes_as(*blob_local, "application/sieve", bytes), @@ -120,6 +161,8 @@ pub fn reconcile( } } + update_batch(net, ty, updates, counts, logger); + if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) { deactivate = true; } @@ -190,22 +233,40 @@ fn target_sieve_extensions(net: &Net) -> Vec { .unwrap_or_default() } +/// A script's bytes after renaming, and the names that were renamed. +type Renamed = (Vec, Vec); + /// The script's bytes with Stalwart's vendor names renamed for an inbuxa -/// target, or `None` when it needs no change. Each rename is logged. -fn renamed_script( - uploader: &Uploader<'_>, - blob_local: i64, - label: &str, - logger: &Logger, -) -> Result>, Error> { +/// target, and the names renamed, or `None` when it needs no change. +fn renamed_script(uploader: &Uploader<'_>, blob_local: i64) -> Result, Error> { let bytes = uploader.bytes(blob_local).map_err(Error::from)?; - Ok(sieve_names::rewrite(&bytes).map(|(out, renamed)| { - for old in &renamed { - let new = old.replacen("vnd.stalwart.", "vnd.inbuxa.", 1); - if logger.enabled(LEVEL_DEFAULT) { - eprintln!("export: SieveScript {label}: renamed {old} to {new}"); - } + Ok(sieve_names::rewrite(&bytes)) +} + +/// Prints each rename made to a script about to be written. +fn log_renames(label: &str, renamed: &[String], logger: &Logger) { + for old in renamed { + let new = old.replacen("vnd.stalwart.", "vnd.inbuxa.", 1); + if logger.enabled(LEVEL_DEFAULT) { + eprintln!("export: SieveScript {label}: renamed {old} to {new}"); } - out - })) + } +} + +/// Whether the target's copy of a script differs from `ours`. A target that +/// reports no blob is taken as different, so ours is written. +fn content_differs(net: &Net, ours: &[u8], target_blob: Option<&String>) -> Result { + let Some(target_blob) = target_blob else { + return Ok(true); + }; + let theirs = blobxfer::download_bytes( + &net.client, + &net.session, + &net.account, + target_blob, + "application/sieve", + "script.sieve", + ) + .map_err(Error::from)?; + Ok(ours != theirs.as_slice()) } diff --git a/src/sync/export/uidtype.rs b/src/sync/export/uidtype.rs index fec9bf6..2ea8804 100644 --- a/src/sync/export/uidtype.rs +++ b/src/sync/export/uidtype.rs @@ -1,15 +1,16 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 John Coffey * * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt::Write as _; -use serde_json::Value; +use serde_json::{Map, Value}; -use super::common::{create_batch, jid, target_query_get}; +use super::common::{create_batch, jid, target_query_get, update_batch}; use super::{Maps, Net, Plan, Uploader}; use crate::error::Error; use crate::logging::Logger; @@ -42,10 +43,10 @@ pub fn reconcile( logger: &Logger, ) -> Result { let targets = target_query_get(net, ty, None).map_err(Error::from)?; - let mut by_uid: std::collections::HashMap = std::collections::HashMap::new(); + let mut by_uid: HashMap = HashMap::new(); for t in &targets { if let (Some(uid), Some(id)) = (target_uid(t), jid(t)) { - by_uid.entry(uid).or_insert(id); + by_uid.entry(uid).or_insert((id, t)); } } @@ -76,12 +77,23 @@ pub fn reconcile( }; let mut matched_uids: HashSet = HashSet::new(); + let mut updates: Vec<(String, Value)> = Vec::new(); let blobs = Uploader::new(net, &ctx.conn); for (local, uid) in &rows { - if let Some(tid) = by_uid.get(uid) { + if let Some((tid, existing)) = by_uid.get(uid) { maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone())); matched_uids.insert(uid.clone()); - counts.skipped += 1; + match build_wire(ctx, ty, *local, maps, &blobs) { + Ok(wire) => match changed_properties(&wire, existing) { + Some(patch) => updates.push((tid.clone(), patch)), + None => counts.skipped += 1, + }, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!("{} not compared: {e}", describe(ty, *local, uid))); + counts.skipped += 1; + } + } continue; } let cid = format!("c{local}"); @@ -123,6 +135,8 @@ pub fn reconcile( } } + update_batch(net, ty, updates, counts, logger); + let objs: Vec = targets .iter() .filter_map(|t| { @@ -172,3 +186,80 @@ fn build_wire( calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from) } } + +/// An `updated` value as a point in time, so that offsets and fractional +/// seconds compare as the same instant. `None` if it does not parse. +fn parse_updated(s: &str) -> Option { + time::OffsetDateTime::parse(s, &time::format_description::well_known::Rfc3339).ok() +} + +/// The update that makes `target` match the archive's `wire` object, or +/// `None` when nothing changed. When both carry `updated`, it decides: the +/// archive's copy wins only if it is newer. Otherwise each property the +/// archive writes is compared, and those that differ are sent whole. +fn changed_properties(wire: &Value, target: &Value) -> Option { + let wire = wire.as_object()?; + let stamp = |v: Option<&Value>| v.and_then(Value::as_str).and_then(parse_updated); + if let (Some(ours), Some(theirs)) = (stamp(wire.get("updated")), stamp(target.get("updated"))) + && ours <= theirs + { + return None; + } + let mut patch = Map::new(); + for (k, v) in wire { + if k == "uid" || k == "id" { + continue; + } + if target.get(k) != Some(v) { + patch.insert(k.clone(), v.clone()); + } + } + (!patch.is_empty()).then_some(Value::Object(patch)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn unchanged_object_needs_no_update() { + let wire = json!({"uid": "u", "name": {"full": "Ann"}, "addressBookIds": {"A": true}}); + let target = json!({"id": "T", "uid": "u", "name": {"full": "Ann"}, + "addressBookIds": {"A": true}, "extra": 1}); + assert_eq!(changed_properties(&wire, &target), None); + } + + #[test] + fn changed_properties_are_sent_whole() { + let wire = json!({"uid": "u", "name": {"full": "Ann B"}, "addressBookIds": {"A": true}}); + let target = json!({"id": "T", "uid": "u", "name": {"full": "Ann"}, + "addressBookIds": {"A": true}}); + assert_eq!( + changed_properties(&wire, &target), + Some(json!({"name": {"full": "Ann B"}})) + ); + } + + #[test] + fn updated_compares_instants_not_strings() { + let target = json!({"uid": "u", "title": "old", "updated": "2026-01-02T00:00:00Z"}); + let same_instant = json!({"uid": "u", "title": "new", + "updated": "2026-01-02T01:00:00.000+01:00"}); + assert_eq!(changed_properties(&same_instant, &target), None); + let later = json!({"uid": "u", "title": "new", "updated": "2026-01-02T00:00:00.5Z"}); + assert!(changed_properties(&later, &target).is_some()); + } + + #[test] + fn updated_decides_when_both_sides_carry_it() { + let target = json!({"uid": "u", "title": "old", "updated": "2026-01-02T00:00:00Z"}); + let older = json!({"uid": "u", "title": "new", "updated": "2026-01-01T00:00:00Z"}); + assert_eq!(changed_properties(&older, &target), None, "target is newer"); + let newer = json!({"uid": "u", "title": "new", "updated": "2026-01-03T00:00:00Z"}); + assert_eq!( + changed_properties(&newer, &target), + Some(json!({"title": "new", "updated": "2026-01-03T00:00:00Z"})) + ); + } +} diff --git a/tests/mock_sync.rs b/tests/mock_sync.rs index 52a04c7..e8b2dbc 100644 --- a/tests/mock_sync.rs +++ b/tests/mock_sync.rs @@ -2575,7 +2575,7 @@ fn export_archive_read_failure_while_inlining_exits_seven() { } #[test] -fn export_sieve_script_matches_by_name_not_content() { +fn export_sieve_script_matched_by_name_is_updated_when_its_content_differs() { let mut server = mockito::Server::new(); let base = server.url(); let api = "/jmap/api"; @@ -2615,9 +2615,125 @@ fn export_sieve_script_matches_by_name_not_content() { ) .expect(1) .create(); - let no_download = server + let download = server .mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into())) - .with_body(b"unused".as_slice()) + .with_body(b"keep;\n".as_slice()) + .expect(1) + .create(); + let upload = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UPN"}).to_string()) + .expect(2) + .create(); + let update = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("SieveScript/set".into()), + Matcher::Regex("\"update\":\\{\"S1\":\\{\"blobId\":\"UPN\"".into()), + ])) + .with_body( + json!({"methodResponses":[["SieveScript/set",{"accountId":"w", + "updated":{"S1":null}},"s"]]}) + .to_string(), + ) + .expect(1) + .create(); + let create = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("SieveScript/set".into()), + Matcher::Regex("reject".into()), + ])) + .with_body( + json!({"methodResponses":[["SieveScript/set",{"accountId":"w", + "created":{"c2":{"id":"S2"}}},"s"]]}) + .to_string(), + ) + .expect(1) + .create(); + let _activate = server + .mock("POST", api) + .match_body(Matcher::Regex("onSuccessActivateScript".into())) + .with_body( + json!({"methodResponses":[["SieveScript/set",{"accountId":"w"},"a"]]}).to_string(), + ) + .expect(1) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("export"); + upload.assert(); + create.assert(); + download.assert(); + update.assert(); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "SieveScript") + .map(|(_, c)| c.clone()) + .expect("sieve counts"); + assert_eq!( + counts.updated, 1, + "the name-matched script gets the archive's content" + ); + assert_eq!(counts.skipped, 0); + assert_eq!(counts.created, 1, "the unmatched name is created"); + assert_eq!(counts.failed, 0); + let _ = std::fs::remove_file(&archive); +} + +#[test] +fn export_sieve_script_matched_by_name_with_the_same_content_is_left_alone() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + let keepall_local = b"require [\"fileinto\"];\nkeep;\n"; + let reject_local = b"require [\"reject\"];\nreject \"go away\";\n"; + { + let conn = db::init::open(&archive).unwrap(); + let blob1 = db::blobs::intern_blob(&conn, keepall_local).unwrap(); + let blob2 = db::blobs::intern_blob(&conn, reject_local).unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'keepall',1,?1)", + rusqlite::params![blob1], + ) + .unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (2,'reject',0,?1)", + rusqlite::params![blob2], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _g = server + .mock("POST", api) + .match_body(Matcher::Regex("SieveScript/get".into())) + .with_body( + json!({"methodResponses":[["SieveScript/get",{"accountId":"w","list":[ + {"id":"S1","name":"keepall","isActive":false,"blobId":"BSRV"} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let download = server + .mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into())) + .with_body(keepall_local.as_slice()) + .expect(1) + .create(); + let no_update = server + .mock("POST", api) + .match_body(Matcher::Regex("\"update\"".into())) .expect(0) .create(); let upload = server @@ -2654,7 +2770,8 @@ fn export_sieve_script_matches_by_name_not_content() { .expect("export"); upload.assert(); create.assert(); - no_download.assert(); + download.assert(); + no_update.assert(); let counts = summary .per_type .iter() @@ -2663,8 +2780,9 @@ fn export_sieve_script_matches_by_name_not_content() { .expect("sieve counts"); assert_eq!( counts.skipped, 1, - "name-matched script is skipped even though its content differs from the target" + "same name and same content: nothing to do" ); + assert_eq!(counts.updated, 0); assert_eq!(counts.created, 1, "the unmatched name is created"); assert_eq!(counts.failed, 0); let _ = std::fs::remove_file(&archive); @@ -4900,3 +5018,285 @@ fn export_different_messages_sharing_a_message_id_are_not_merged() { no_set.assert(); let _ = std::fs::remove_file(&archive); } + +#[test] +fn export_rerun_carries_a_read_flag_set_at_the_source() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + let _setup = two_folder_archive_and_target(&mut server, &archive); + insert_email_copy(&archive, ONE_MESSAGE, 1); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute("UPDATE emails SET keywords='[\"$seen\"]'", []) + .unwrap(); + } + + let _eq = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/query".into())) + .with_body( + json!({"methodResponses":[["Email/query",{"accountId":"w","ids":["X1"]},"q"]]}) + .to_string(), + ) + .expect(1) + .create(); + let _eg = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/get".into())) + .with_body( + json!({"methodResponses":[["Email/get",{"accountId":"w","list":[ + {"id":"X1","messageId":["both@h"],"size":ONE_MESSAGE.len(), + "mailboxIds":{"T1":true},"keywords":{"$flagged":true}} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let set = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("Email/set".into()), + Matcher::Regex("\"keywords/\\$seen\":true".into()), + Matcher::Regex("\"keywords/\\$flagged\":null".into()), + ])) + .with_body( + json!({"methodResponses":[["Email/set",{"accountId":"w","updated":{"X1":null}},"s"]]}) + .to_string(), + ) + .expect(1) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]), + ) + .expect("export"); + let email = email_counts(&summary); + assert_eq!(email.updated, 1); + assert_eq!(email.created, 0); + assert_eq!(email.failed, 0); + set.assert(); + let _ = std::fs::remove_file(&archive); +} + +fn contact_rerun(local_updated: &str, updates_sent: usize) -> inbuxa_migrate::sync::TypeCounts { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO address_books (id,name,description,is_default) VALUES (1,'Personal',NULL,1)", + [], + ) + .unwrap(); + let card = json!({"@type":"Card","version":"1.0","uid":"u1", + "name":{"full":"Ann Brown"},"updated":local_updated}); + conn.execute( + "INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (1,'u1','[1]',?1)", + rusqlite::params![card.to_string()], + ) + .unwrap(); + } + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _abg = server + .mock("POST", api) + .match_body(Matcher::Regex("AddressBook/get".into())) + .with_body( + json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[ + {"id":"P","name":"Personal","isDefault":true,"myRights":{"mayDelete":false}} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect_at_least(1) + .create(); + let _term = anchor_terminator(&mut server, api, "ContactCard"); + let _cq = server + .mock("POST", api) + .match_body(Matcher::Regex("ContactCard/query".into())) + .with_body( + json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":["C1"]},"q"]]}) + .to_string(), + ) + .expect(1) + .create(); + let _cg = server + .mock("POST", api) + .match_body(Matcher::Regex("ContactCard/get".into())) + .with_body( + json!({"methodResponses":[["ContactCard/get",{"accountId":"w","list":[ + {"id":"C1","@type":"Card","version":"1.0","uid":"u1", + "name":{"full":"Ann"},"addressBookIds":{"P":true}, + "updated":"2026-02-01T00:00:00Z"} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let update = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("ContactCard/set".into()), + Matcher::Regex("\"update\"".into()), + Matcher::Regex("Ann Brown".into()), + ])) + .with_body( + json!({"methodResponses":[["ContactCard/set",{"accountId":"w","updated":{"C1":null}},"s"]]}) + .to_string(), + ) + .expect(updates_sent) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects( + &base, + vec![ObjectType::AddressBook, ObjectType::ContactCard], + ), + ) + .expect("export"); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "ContactCard") + .map(|(_, c)| c.clone()) + .expect("contact counts"); + update.assert(); + let _ = std::fs::remove_file(&archive); + counts +} + +#[test] +fn export_rerun_updates_a_contact_edited_at_the_source() { + let counts = contact_rerun("2026-03-01T00:00:00Z", 1); + assert_eq!(counts.updated, 1, "the newer archive copy is written"); + assert_eq!(counts.failed, 0); +} + +#[test] +fn export_rerun_leaves_a_contact_alone_when_the_target_is_newer() { + let counts = contact_rerun("2026-01-01T00:00:00Z", 0); + assert_eq!( + counts.updated, 0, + "the target's copy is newer, so nothing is sent" + ); + assert_eq!(counts.skipped, 1); +} + +fn event_rerun(local_updated: &str, updates_sent: usize) -> inbuxa_migrate::sync::TypeCounts { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO calendars (id,name,is_default) VALUES (1,'Work',1)", + [], + ) + .unwrap(); + let event = json!({"@type":"Event","uid":"ev1","title":"Planning, moved", + "start":"2026-03-02T10:00:00","duration":"PT1H", + "updated":local_updated}); + conn.execute( + "INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data) + VALUES (1,'[1]',0,0,?1)", + rusqlite::params![event.to_string()], + ) + .unwrap(); + } + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _calg = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/get".into())) + .with_body( + json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[ + {"id":"K","name":"Work","isDefault":true,"myRights":{"mayDelete":false}} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect_at_least(1) + .create(); + let _term = anchor_terminator(&mut server, api, "CalendarEvent"); + let _eq = server + .mock("POST", api) + .match_body(Matcher::Regex("CalendarEvent/query".into())) + .with_body( + json!({"methodResponses":[["CalendarEvent/query",{"accountId":"w","ids":["E1"]},"q"]]}) + .to_string(), + ) + .expect(1) + .create(); + let _eg = server + .mock("POST", api) + .match_body(Matcher::Regex("CalendarEvent/get".into())) + .with_body( + json!({"methodResponses":[["CalendarEvent/get",{"accountId":"w","list":[ + {"id":"E1","@type":"Event","uid":"ev1","title":"Planning", + "start":"2026-03-01T10:00:00","duration":"PT1H", + "calendarIds":{"K":true},"isDraft":false,"useDefaultAlerts":false, + "updated":"2026-02-01T00:00:00Z"} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let update = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("CalendarEvent/set".into()), + Matcher::Regex("\"update\"".into()), + Matcher::Regex("Planning, moved".into()), + ])) + .with_body( + json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w","updated":{"E1":null}},"s"]]}) + .to_string(), + ) + .expect(updates_sent) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::Calendar, ObjectType::CalendarEvent]), + ) + .expect("export"); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "CalendarEvent") + .map(|(_, c)| c.clone()) + .expect("event counts"); + update.assert(); + let _ = std::fs::remove_file(&archive); + counts +} + +#[test] +fn export_rerun_updates_an_event_moved_at_the_source() { + let counts = event_rerun("2026-03-01T00:00:00Z", 1); + assert_eq!(counts.updated, 1, "the newer archive copy is written"); + assert_eq!(counts.failed, 0); +} + +#[test] +fn export_rerun_leaves_an_event_alone_when_nothing_is_newer() { + let counts = event_rerun("2026-02-01T01:00:00+01:00", 0); + assert_eq!( + counts.updated, 0, + "same instant as the target's, written another way" + ); + assert_eq!(counts.skipped, 1); +}