From 34981475f2c49b969d86ec8f01bb5fb88506db5e Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 16 Nov 2026 23:52:18 -0700 Subject: [PATCH] Export: match a message the target holds under its own key A message with two Message-ID headers is keyed by both in the archive, while a JMAP server's messageId reflects one, so export never paired it and imported it again on every run (shown live on inbuxa 27.1-rc.4). While target emails are left unpaired, a batch about to be imported is parsed by the target first (Email/parse) and each message it already holds under that key is matched, with its keywords and folders synced like any other match. A target that refuses Email/parse is not asked again; any other failure leaves that batch unchecked, imported as before. The parsed key also settles an import that ended without a clear answer. Upstream fixed the same case in v1.0.17 (#86); written here on our own code and tests. --- CHANGELOG.md | 12 +++ src/sync/export/email.rs | 175 ++++++++++++++++++++++++++++-- tests/integration_inbuxa.rs | 58 ++++++++++ tests/mock_sync.rs | 207 ++++++++++++++++++++++++++++++++++++ 4 files changed, 443 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fffbb4..6e96a13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Fixed +- `export` no longer imports a message again on every run when the archive + and the target key it differently, as with a message carrying two + `Message-ID` headers: the archive keeps both, a JMAP server's `messageId` + one. While target emails are left unpaired, a message about to be imported + is first parsed by the target (`Email/parse`), and one it already holds is + matched instead, with its keywords and folders brought in line like any + other match. A target without `Email/parse` gets the message imported as + before. The same key settles a batch whose import ended without a clear + answer, so one that did arrive is not sent twice. Upstream fixed the same + case in v1.0.17; this is our own fix. + ## [27.1-rc.14] The fourteenth release candidate of the 27.1 series: POP3, Fastmail, Zoho and diff --git a/src/sync/export/email.rs b/src/sync/export/email.rs index 4b19947..01d7bc2 100644 --- a/src/sync/export/email.rs +++ b/src/sync/export/email.rs @@ -20,7 +20,9 @@ use crate::jmap::retry::MethodCallKind; use crate::jmap::wire::JmapId; use crate::logging::Logger; use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email}; -use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json}; +use crate::sync::keys::{ + EmailIndex, EmailKey, email_index, email_key, email_keys, index_from_json, +}; use crate::sync::progress::Progress; use crate::sync::{Context, TypeCounts}; use crate::types::ObjectType; @@ -107,6 +109,17 @@ pub fn reconcile( None => creates.push(i), } } + let mut paired = vec![false; targets.len()]; + for t in pairs.iter().flatten() { + paired[*t] = true; + } + let mut unpaired = Unpaired { + targets: &targets, + keys: &target_keys, + free: (0..targets.len()).filter(|t| !paired[*t]).collect(), + migrated: &migrated, + parse: true, + }; let mut progress = Progress::new("export: Email", creates.len() as u64, logger); import_units( net, @@ -114,6 +127,8 @@ pub fn reconcile( maps, &units, &creates, + &mut unpaired, + &mut updates, counts, logger, &mut progress, @@ -170,12 +185,100 @@ fn target_emails(net: &Net) -> Result<(Vec, Vec), JmapErr /// without a clear answer leaves few messages to check. const IMPORT_BATCH_CAP: usize = 50; -/// One message ready to import: its creation id, its place in `units`, and -/// the `Email/import` entry. +/// One message ready to import: its creation id, its place in `units`, the +/// `Email/import` entry, and the key the target gave it when it parsed the +/// uploaded bytes, if it was asked. struct Pending { cid: String, unit: usize, item: Value, + parsed: Option, +} + +/// The target emails no archive message paired with by its stored key. A +/// message the target already holds can still be among them under another +/// key: the archive keys a message by every `Message-ID` header it has, while +/// the target's `messageId` reflects one, and the two parsers may differ in +/// other ways too. Before such a message is imported, the target parses the +/// uploaded bytes and the key it computes is looked up here. +struct Unpaired<'t> { + targets: &'t [TargetEmail], + keys: &'t [EmailKey], + free: Vec, + migrated: &'t HashSet, + /// False once the target has refused `Email/parse`: not asked again. + parse: bool, +} + +impl Unpaired<'_> { + /// Takes the unpaired target email with `key`, the one of `size` first. + fn take(&mut self, key: &EmailKey, size: Option) -> Option { + let same_key = |t: &usize| &self.keys[*t] == key; + let pos = self + .free + .iter() + .position(|t| same_key(t) && size.is_some() && self.targets[*t].size == size) + .or_else(|| self.free.iter().position(same_key))?; + Some(self.free.remove(pos)) + } +} + +/// The most blobs one `Email/parse` call names, within `maxObjectsInGet`. +const PARSE_BATCH_CAP: usize = 50; +const PARSE_PROPERTIES: [&str; 5] = ["messageId", "from", "subject", "sentAt", "to"]; + +/// The key the target computes for each of `blobs`, by blob id. A blob the +/// target could not parse has none. If the target refuses the method, it is +/// not asked again this run; any other failure leaves this batch unchecked. +/// Either way the messages are then imported as before. +fn parsed_keys( + net: &Net, + blobs: &[&str], + unpaired: &mut Unpaired<'_>, + logger: &Logger, +) -> HashMap { + let mut out = HashMap::new(); + let per_call = (net.limits.max_objects_in_get as usize).clamp(1, PARSE_BATCH_CAP); + for chunk in blobs.chunks(per_call) { + let mut req = Request::new(); + req.call( + "Email/parse", + json!({ + "accountId": net.account, + "blobIds": chunk, + "properties": PARSE_PROPERTIES, + }), + "p", + ); + let answer = req.send(&net.client, &net.api).and_then(|resp| { + let mr = resp.first()?.clone(); + check_method_error(&mr)?; + Ok(mr) + }); + let mr = match answer { + Ok(mr) => mr, + Err(e @ JmapError::Method { .. }) => { + logger.warn(&format!( + "the target refused Email/parse ({e}); messages whose stored match key differs from the target's are imported without that check" + )); + unpaired.parse = false; + return out; + } + Err(e) => { + logger.warn(&format!( + "Email/parse failed ({e}); {} message(s) are imported without checking the target for them under another key", + chunk.len() + )); + continue; + } + }; + if let Some(parsed) = mr.args.get("parsed").and_then(Value::as_object) { + for (blob, email) in parsed { + out.insert(blob.clone(), email_key(&server_index(email))); + } + } + } + out } /// Writes the messages the target does not have yet. They go in batches of @@ -191,6 +294,8 @@ fn import_units( maps: &Maps, units: &[Unit], creates: &[usize], + unpaired: &mut Unpaired<'_>, + updates: &mut Vec<(String, Value)>, counts: &mut TypeCounts, logger: &Logger, progress: &mut Progress, @@ -232,6 +337,7 @@ fn import_units( cid, unit: i, item: import_item(blob.0, mids, build_keywords(row), &row.received_at), + parsed: None, }), Err(e) => { logger.warn(&format!( @@ -251,6 +357,9 @@ fn import_units( if net.dry_run { counts.created += pending.len() as u64; } else { + let pending = match_by_target_parse( + net, uploader, maps, units, pending, unpaired, updates, counts, logger, + ); send_batch( net, uploader, @@ -269,6 +378,50 @@ fn import_units( } } +/// Asks the target to parse the batch's uploaded messages while unpaired +/// target emails remain, and keeps back each one it already holds under the +/// key it computes: that email is brought in line like any other match. The +/// rest are returned to import, each with the target's key for settling. +#[allow(clippy::too_many_arguments)] +fn match_by_target_parse( + net: &Net, + uploader: &Uploader, + maps: &Maps, + units: &[Unit], + pending: Vec, + unpaired: &mut Unpaired<'_>, + updates: &mut Vec<(String, Value)>, + counts: &mut TypeCounts, + logger: &Logger, +) -> Vec { + if !unpaired.parse || unpaired.free.is_empty() || pending.is_empty() { + return pending; + } + let blobs: Vec<&str> = pending + .iter() + .filter_map(|p| p.item.get("blobId").and_then(Value::as_str)) + .collect(); + let mut keys = parsed_keys(net, &blobs, unpaired, logger); + let mut out = Vec::with_capacity(pending.len()); + for mut p in pending { + let blob = p.item.get("blobId").and_then(Value::as_str).unwrap_or(""); + p.parsed = keys.remove(blob); + let row = &units[p.unit].row; + let found = p + .parsed + .as_ref() + .and_then(|k| unpaired.take(k, uploader.blob_len(row.blob_local_id))); + match found { + Some(t) => match email_patch(row, &unpaired.targets[t], maps, unpaired.migrated) { + Some(patch) => updates.push((unpaired.targets[t].id.clone(), patch)), + None => counts.skipped += 1, + }, + None => out.push(p), + } + } + out +} + /// Sends one `Email/import` for `batch` and counts each message's outcome. /// A request too large for the server is split in two; a method error that /// rejects the whole call is retried one message at a time, so the one at @@ -406,12 +559,16 @@ fn settle_unclear( return; } }; - let keys: Vec = email_keys( - &unclear - .iter() - .map(|p| index_from_json(&units[p.unit].row.message_match)) - .collect::>(), - ); + // The target's own key where it parsed the message: one that arrived is + // found under it even when the stored key differs. + let keys: Vec = unclear + .iter() + .map(|p| { + p.parsed + .clone() + .unwrap_or_else(|| email_key(&index_from_json(&units[p.unit].row.message_match))) + }) + .collect(); let sizes: Vec> = unclear .iter() .map(|p| uploader.blob_len(units[p.unit].row.blob_local_id)) diff --git a/tests/integration_inbuxa.rs b/tests/integration_inbuxa.rs index 66f1635..305f93f 100644 --- a/tests/integration_inbuxa.rs +++ b/tests/integration_inbuxa.rs @@ -454,3 +454,61 @@ fn files_go_into_inbuxa_in_its_own_shape_and_come_back() { ] ); } + +#[test] +#[ignore = "requires Docker"] +fn a_message_with_two_message_ids_is_not_exported_twice() { + let server = Inbuxa::start().expect("start inbuxa"); + let dir = TempDir::new().unwrap(); + let maildir = dir.path().join("mail"); + let archive = dir.path().join("archive.sqlite"); + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(maildir.join(sub)).unwrap(); + } + // The archive keys it by both Message-IDs; inbuxa's messageId has one. + fs::write( + maildir.join("cur/1:2,S"), + "From: x@example.org\r\nTo: y@example.org\r\nSubject: Twice\r\n\ + Message-ID: \r\nMessage-ID: \r\n\ + Date: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n", + ) + .unwrap(); + + let run = |args: &[&str]| { + let out = Command::new(BIN) + .args(args) + .env("INBUXA_MIGRATE_API_KEY", &server.token) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + assert!(out.status.success(), "{args:?}: {stderr}"); + String::from_utf8_lossy(&out.stdout).into_owned() + }; + run(&[ + "import", + "maildir", + maildir.to_str().unwrap(), + archive.to_str().unwrap(), + ]); + let export = || { + run(&[ + "export", + "--url", + &server.base_url, + "--auth-bearer", + "--account-name", + ADMIN_EMAIL, + "--objects", + "mailbox,email", + archive.to_str().unwrap(), + ]) + }; + let first = export(); + assert!(first.contains("Email: created=1"), "{first}"); + let second = export(); + assert!(second.contains("Email: created=0"), "{second}"); + assert_eq!( + emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD), + [("Twice".to_owned(), 1, vec!["$seen".to_owned()])] + ); +} diff --git a/tests/mock_sync.rs b/tests/mock_sync.rs index fdc7364..4aa0b07 100644 --- a/tests/mock_sync.rs +++ b/tests/mock_sync.rs @@ -6204,3 +6204,210 @@ fn export_file_nodes_in_the_targets_own_shape() { let _ = std::fs::remove_file(&archive); } } + +/// A message with two Message-ID headers: the archive keys it by both, while a +/// JMAP server's `messageId` (RFC 8621 `header:Message-ID:asMessageIds`) +/// reflects one header. The stored key and the target's then differ, and +/// only the target's own parse of the bytes can tell that it already has it. +const TWO_MESSAGE_IDS: &str = + "From: a@x\r\nSubject: s\r\nMessage-ID: \r\nMessage-ID: \r\n\r\nbody"; + +struct DivergentKeyExport { + /// The `Email/parse` answer, or `None` when the test expects no call. + parse: Option, + /// The local copy's keywords, as stored (a JSON array). + local_keywords: &'static str, + imports: usize, + sets: usize, +} + +fn export_against_a_divergent_target_key(t: DivergentKeyExport) -> sync::TypeCounts { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')", + [], + ) + .unwrap(); + let blob = db::blobs::intern_blob(&conn, TWO_MESSAGE_IDS.as_bytes()).unwrap(); + let idx = + inbuxa_migrate::sync::emailmeta::email_index_from_blob(TWO_MESSAGE_IDS.as_bytes()); + assert_eq!( + idx.mids, + ["a@h", "b@h"], + "the archive keys by every Message-ID" + ); + conn.execute( + "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match) + VALUES (?1,'2020-01-01T00:00:00Z','[1]',?2,?3)", + rusqlite::params![ + blob, + t.local_keywords, + inbuxa_migrate::sync::keys::index_to_json(&idx) + ], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _mbterm = anchor_terminator(&mut server, api, "Mailbox"); + let _emterm = anchor_terminator(&mut server, api, "Email"); + let _mq = server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/query".into())) + .with_body( + json!({"methodResponses":[["Mailbox/query", + {"accountId":"w","ids":["T1"]},"q"]]}) + .to_string(), + ) + .expect_at_least(1) + .create(); + let _mg = server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/get".into())) + .with_body( + json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[ + {"id":"T1","name":"Inbox","role":"inbox","parentId":null,"myRights":{"mayDelete":true}} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect_at_least(1) + .create(); + let _eq = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/query".into())) + .with_body( + json!({"methodResponses":[["Email/query", + {"accountId":"w","ids":["X1"]},"q"]]}) + .to_string(), + ) + .expect(1) + .create(); + let _eg = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/get".into())) + .with_body( + json!({"methodResponses":[["Email/get",{"accountId":"w","list":[ + {"id":"X1","messageId":["b@h"],"size":TWO_MESSAGE_IDS.len(), + "mailboxIds":{"T1":true},"keywords":{}} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UP"}).to_string()) + .expect_at_most(1) + .create(); + let parse = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("Email/parse".into()), + Matcher::Regex(r#""blobIds":\["UP"\]"#.into()), + ])) + .with_body( + json!({ "methodResponses": [t.parse.clone().unwrap_or(Value::Null)] }).to_string(), + ) + .expect(usize::from(t.parse.is_some())) + .create(); + let import = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/import".into())) + .with_body( + json!({"methodResponses":[["Email/import",{"accountId":"w", + "created":{"e1":{"id":"Y1","blobId":"UP","threadId":"t","size":10}}},"i"]]}) + .to_string(), + ) + .expect(t.imports) + .create(); + let set = server + .mock("POST", api) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("Email/set".into()), + Matcher::Regex(r#""keywords/\$seen":true"#.into()), + ])) + .with_body( + json!({"methodResponses":[["Email/set",{"accountId":"w","updated":{"X1":null}},"s"]]}) + .to_string(), + ) + .expect(t.sets) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]), + ) + .expect("export"); + parse.assert(); + import.assert(); + set.assert(); + let _ = std::fs::remove_file(&archive); + summary + .per_type + .iter() + .find(|(t, _)| *t == "Email") + .map(|(_, c)| c.clone()) + .expect("email counts") +} + +fn parsed_as_b() -> Value { + json!(["Email/parse", {"accountId":"w","parsed":{"UP":{ + "messageId":["b@h"],"from":[{"name":null,"email":"a@x"}],"subject":"s","sentAt":null,"to":null}}, + "notParsable":[],"notFound":[]}, "p0"]) +} + +#[test] +fn export_email_the_target_already_holds_under_its_own_key_is_not_imported_again() { + let email = export_against_a_divergent_target_key(DivergentKeyExport { + parse: Some(parsed_as_b()), + local_keywords: "[]", + imports: 0, + sets: 0, + }); + assert_eq!((email.created, email.skipped, email.failed), (0, 1, 0)); +} + +#[test] +fn export_email_matched_by_the_target_parse_gets_its_keywords_synced() { + let email = export_against_a_divergent_target_key(DivergentKeyExport { + parse: Some(parsed_as_b()), + local_keywords: r#"["$seen"]"#, + imports: 0, + sets: 1, + }); + assert_eq!((email.created, email.failed), (0, 0)); +} + +#[test] +fn export_email_is_imported_when_the_target_cannot_parse() { + let email = export_against_a_divergent_target_key(DivergentKeyExport { + parse: Some(json!(["error", {"type":"unknownMethod"}, "p0"])), + local_keywords: "[]", + imports: 1, + sets: 0, + }); + assert_eq!((email.created, email.skipped, email.failed), (1, 0, 0)); +} + +#[test] +fn export_email_parsed_as_a_different_message_is_imported() { + let email = export_against_a_divergent_target_key(DivergentKeyExport { + parse: Some(json!(["Email/parse", {"accountId":"w","parsed":{"UP":{ + "messageId":["a@h"],"from":null,"subject":"s","sentAt":null,"to":null}}, + "notParsable":[],"notFound":[]}, "p0"])), + local_keywords: "[]", + imports: 1, + sets: 0, + }); + assert_eq!((email.created, email.skipped, email.failed), (1, 0, 0)); +} -- 2.54.0