Export: match a message the target holds under its own key #87

Merged
jcoffey-dev merged 2 commits from fix/export-parse-rematch into main 2026-11-17 15:50:32 +00:00
4 changed files with 443 additions and 9 deletions

No files matched your search

+12
View File
@@ -13,6 +13,18 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
mail-parser 1.0 and rusqlite 0.43. Upstream's new console banner is left
out.
### Fixed
- `export` no longer imports a message again on every run when the archive
and the target key it differently, as with a message carrying two
`Message-ID` headers: the archive keeps both, a JMAP server's `messageId`
one. While target emails are left unpaired, a message about to be imported
is first parsed by the target (`Email/parse`), and one it already holds is
matched instead, with its keywords and folders brought in line like any
other match. A target without `Email/parse` gets the message imported as
before. The same key settles a batch whose import ended without a clear
answer, so one that did arrive is not sent twice. Upstream fixed the same
case in v1.0.17; this is our own fix.
## [27.1-rc.14]
The fourteenth release candidate of the 27.1 series: POP3, Fastmail, Zoho and
+166 -9
View File
@@ -20,7 +20,9 @@ use crate::jmap::retry::MethodCallKind;
use crate::jmap::wire::JmapId;
use crate::logging::Logger;
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json};
use crate::sync::keys::{
EmailIndex, EmailKey, email_index, email_key, email_keys, index_from_json,
};
use crate::sync::progress::Progress;
use crate::sync::{Context, TypeCounts};
use crate::types::ObjectType;
@@ -107,6 +109,17 @@ pub fn reconcile(
None => creates.push(i),
}
}
let mut paired = vec![false; targets.len()];
for t in pairs.iter().flatten() {
paired[*t] = true;
}
let mut unpaired = Unpaired {
targets: &targets,
keys: &target_keys,
free: (0..targets.len()).filter(|t| !paired[*t]).collect(),
migrated: &migrated,
parse: true,
};
let mut progress = Progress::new("export: Email", creates.len() as u64, logger);
import_units(
net,
@@ -114,6 +127,8 @@ pub fn reconcile(
maps,
&units,
&creates,
&mut unpaired,
&mut updates,
counts,
logger,
&mut progress,
@@ -170,12 +185,100 @@ fn target_emails(net: &Net) -> Result<(Vec<TargetEmail>, Vec<EmailKey>), JmapErr
/// without a clear answer leaves few messages to check.
const IMPORT_BATCH_CAP: usize = 50;
/// One message ready to import: its creation id, its place in `units`, and
/// the `Email/import` entry.
/// One message ready to import: its creation id, its place in `units`, the
/// `Email/import` entry, and the key the target gave it when it parsed the
/// uploaded bytes, if it was asked.
struct Pending {
cid: String,
unit: usize,
item: Value,
parsed: Option<EmailKey>,
}
/// The target emails no archive message paired with by its stored key. A
/// message the target already holds can still be among them under another
/// key: the archive keys a message by every `Message-ID` header it has, while
/// the target's `messageId` reflects one, and the two parsers may differ in
/// other ways too. Before such a message is imported, the target parses the
/// uploaded bytes and the key it computes is looked up here.
struct Unpaired<'t> {
targets: &'t [TargetEmail],
keys: &'t [EmailKey],
free: Vec<usize>,
migrated: &'t HashSet<String>,
/// False once the target has refused `Email/parse`: not asked again.
parse: bool,
}
impl Unpaired<'_> {
/// Takes the unpaired target email with `key`, the one of `size` first.
fn take(&mut self, key: &EmailKey, size: Option<u64>) -> Option<usize> {
let same_key = |t: &usize| &self.keys[*t] == key;
let pos = self
.free
.iter()
.position(|t| same_key(t) && size.is_some() && self.targets[*t].size == size)
.or_else(|| self.free.iter().position(same_key))?;
Some(self.free.remove(pos))
}
}
/// The most blobs one `Email/parse` call names, within `maxObjectsInGet`.
const PARSE_BATCH_CAP: usize = 50;
const PARSE_PROPERTIES: [&str; 5] = ["messageId", "from", "subject", "sentAt", "to"];
/// The key the target computes for each of `blobs`, by blob id. A blob the
/// target could not parse has none. If the target refuses the method, it is
/// not asked again this run; any other failure leaves this batch unchecked.
/// Either way the messages are then imported as before.
fn parsed_keys(
net: &Net,
blobs: &[&str],
unpaired: &mut Unpaired<'_>,
logger: &Logger,
) -> HashMap<String, EmailKey> {
let mut out = HashMap::new();
let per_call = (net.limits.max_objects_in_get as usize).clamp(1, PARSE_BATCH_CAP);
for chunk in blobs.chunks(per_call) {
let mut req = Request::new();
req.call(
"Email/parse",
json!({
"accountId": net.account,
"blobIds": chunk,
"properties": PARSE_PROPERTIES,
}),
"p",
);
let answer = req.send(&net.client, &net.api).and_then(|resp| {
let mr = resp.first()?.clone();
check_method_error(&mr)?;
Ok(mr)
});
let mr = match answer {
Ok(mr) => mr,
Err(e @ JmapError::Method { .. }) => {
logger.warn(&format!(
"the target refused Email/parse ({e}); messages whose stored match key differs from the target's are imported without that check"
));
unpaired.parse = false;
return out;
}
Err(e) => {
logger.warn(&format!(
"Email/parse failed ({e}); {} message(s) are imported without checking the target for them under another key",
chunk.len()
));
continue;
}
};
if let Some(parsed) = mr.args.get("parsed").and_then(Value::as_object) {
for (blob, email) in parsed {
out.insert(blob.clone(), email_key(&server_index(email)));
}
}
}
out
}
/// Writes the messages the target does not have yet. They go in batches of
@@ -191,6 +294,8 @@ fn import_units(
maps: &Maps,
units: &[Unit],
creates: &[usize],
unpaired: &mut Unpaired<'_>,
updates: &mut Vec<(String, Value)>,
counts: &mut TypeCounts,
logger: &Logger,
progress: &mut Progress,
@@ -232,6 +337,7 @@ fn import_units(
cid,
unit: i,
item: import_item(blob.0, mids, build_keywords(row), &row.received_at),
parsed: None,
}),
Err(e) => {
logger.warn(&format!(
@@ -251,6 +357,9 @@ fn import_units(
if net.dry_run {
counts.created += pending.len() as u64;
} else {
let pending = match_by_target_parse(
net, uploader, maps, units, pending, unpaired, updates, counts, logger,
);
send_batch(
net,
uploader,
@@ -269,6 +378,50 @@ fn import_units(
}
}
/// Asks the target to parse the batch's uploaded messages while unpaired
/// target emails remain, and keeps back each one it already holds under the
/// key it computes: that email is brought in line like any other match. The
/// rest are returned to import, each with the target's key for settling.
#[allow(clippy::too_many_arguments)]
fn match_by_target_parse(
net: &Net,
uploader: &Uploader,
maps: &Maps,
units: &[Unit],
pending: Vec<Pending>,
unpaired: &mut Unpaired<'_>,
updates: &mut Vec<(String, Value)>,
counts: &mut TypeCounts,
logger: &Logger,
) -> Vec<Pending> {
if !unpaired.parse || unpaired.free.is_empty() || pending.is_empty() {
return pending;
}
let blobs: Vec<&str> = pending
.iter()
.filter_map(|p| p.item.get("blobId").and_then(Value::as_str))
.collect();
let mut keys = parsed_keys(net, &blobs, unpaired, logger);
let mut out = Vec::with_capacity(pending.len());
for mut p in pending {
let blob = p.item.get("blobId").and_then(Value::as_str).unwrap_or("");
p.parsed = keys.remove(blob);
let row = &units[p.unit].row;
let found = p
.parsed
.as_ref()
.and_then(|k| unpaired.take(k, uploader.blob_len(row.blob_local_id)));
match found {
Some(t) => match email_patch(row, &unpaired.targets[t], maps, unpaired.migrated) {
Some(patch) => updates.push((unpaired.targets[t].id.clone(), patch)),
None => counts.skipped += 1,
},
None => out.push(p),
}
}
out
}
/// Sends one `Email/import` for `batch` and counts each message's outcome.
/// A request too large for the server is split in two; a method error that
/// rejects the whole call is retried one message at a time, so the one at
@@ -406,12 +559,16 @@ fn settle_unclear(
return;
}
};
let keys: Vec<EmailKey> = email_keys(
&unclear
.iter()
.map(|p| index_from_json(&units[p.unit].row.message_match))
.collect::<Vec<_>>(),
);
// The target's own key where it parsed the message: one that arrived is
// found under it even when the stored key differs.
let keys: Vec<EmailKey> = unclear
.iter()
.map(|p| {
p.parsed
.clone()
.unwrap_or_else(|| email_key(&index_from_json(&units[p.unit].row.message_match)))
})
.collect();
let sizes: Vec<Option<u64>> = unclear
.iter()
.map(|p| uploader.blob_len(units[p.unit].row.blob_local_id))
+58
View File
@@ -454,3 +454,61 @@ fn files_go_into_inbuxa_in_its_own_shape_and_come_back() {
]
);
}
#[test]
#[ignore = "requires Docker"]
fn a_message_with_two_message_ids_is_not_exported_twice() {
let server = Inbuxa::start().expect("start inbuxa");
let dir = TempDir::new().unwrap();
let maildir = dir.path().join("mail");
let archive = dir.path().join("archive.sqlite");
for sub in ["cur", "new", "tmp"] {
fs::create_dir_all(maildir.join(sub)).unwrap();
}
// The archive keys it by both Message-IDs; inbuxa's messageId has one.
fs::write(
maildir.join("cur/1:2,S"),
"From: [email protected]\r\nTo: [email protected]\r\nSubject: Twice\r\n\
Message-ID: <[email protected]>\r\nMessage-ID: <[email protected]>\r\n\
Date: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n",
)
.unwrap();
let run = |args: &[&str]| {
let out = Command::new(BIN)
.args(args)
.env("INBUXA_MIGRATE_API_KEY", &server.token)
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
assert!(out.status.success(), "{args:?}: {stderr}");
String::from_utf8_lossy(&out.stdout).into_owned()
};
run(&[
"import",
"maildir",
maildir.to_str().unwrap(),
archive.to_str().unwrap(),
]);
let export = || {
run(&[
"export",
"--url",
&server.base_url,
"--auth-bearer",
"--account-name",
ADMIN_EMAIL,
"--objects",
"mailbox,email",
archive.to_str().unwrap(),
])
};
let first = export();
assert!(first.contains("Email: created=1"), "{first}");
let second = export();
assert!(second.contains("Email: created=0"), "{second}");
assert_eq!(
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
[("Twice".to_owned(), 1, vec!["$seen".to_owned()])]
);
}
+207
View File
@@ -6204,3 +6204,210 @@ fn export_file_nodes_in_the_targets_own_shape() {
let _ = std::fs::remove_file(&archive);
}
}
/// A message with two Message-ID headers: the archive keys it by both, while a
/// JMAP server's `messageId` (RFC 8621 `header:Message-ID:asMessageIds`)
/// reflects one header. The stored key and the target's then differ, and
/// only the target's own parse of the bytes can tell that it already has it.
const TWO_MESSAGE_IDS: &str =
"From: a@x\r\nSubject: s\r\nMessage-ID: <a@h>\r\nMessage-ID: <b@h>\r\n\r\nbody";
struct DivergentKeyExport {
/// The `Email/parse` answer, or `None` when the test expects no call.
parse: Option<Value>,
/// The local copy's keywords, as stored (a JSON array).
local_keywords: &'static str,
imports: usize,
sets: usize,
}
fn export_against_a_divergent_target_key(t: DivergentKeyExport) -> sync::TypeCounts {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
{
let conn = db::init::open(&archive).unwrap();
conn.execute(
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
[],
)
.unwrap();
let blob = db::blobs::intern_blob(&conn, TWO_MESSAGE_IDS.as_bytes()).unwrap();
let idx =
inbuxa_migrate::sync::emailmeta::email_index_from_blob(TWO_MESSAGE_IDS.as_bytes());
assert_eq!(
idx.mids,
["a@h", "b@h"],
"the archive keys by every Message-ID"
);
conn.execute(
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match)
VALUES (?1,'2020-01-01T00:00:00Z','[1]',?2,?3)",
rusqlite::params![
blob,
t.local_keywords,
inbuxa_migrate::sync::keys::index_to_json(&idx)
],
)
.unwrap();
}
let _root = server.mock("GET", "/").with_status(404).create();
let _wk = server
.mock("GET", "/.well-known/jmap")
.with_body(session_body_full(&base))
.expect_at_least(1)
.create();
let _mbterm = anchor_terminator(&mut server, api, "Mailbox");
let _emterm = anchor_terminator(&mut server, api, "Email");
let _mq = server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/query".into()))
.with_body(
json!({"methodResponses":[["Mailbox/query",
{"accountId":"w","ids":["T1"]},"q"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _mg = server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/get".into()))
.with_body(
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
{"id":"T1","name":"Inbox","role":"inbox","parentId":null,"myRights":{"mayDelete":true}}
],"notFound":[]},"g"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _eq = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/query".into()))
.with_body(
json!({"methodResponses":[["Email/query",
{"accountId":"w","ids":["X1"]},"q"]]})
.to_string(),
)
.expect(1)
.create();
let _eg = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/get".into()))
.with_body(
json!({"methodResponses":[["Email/get",{"accountId":"w","list":[
{"id":"X1","messageId":["b@h"],"size":TWO_MESSAGE_IDS.len(),
"mailboxIds":{"T1":true},"keywords":{}}
],"notFound":[]},"g"]]})
.to_string(),
)
.expect(1)
.create();
let _up = server
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
.with_body(json!({"blobId":"UP"}).to_string())
.expect_at_most(1)
.create();
let parse = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("Email/parse".into()),
Matcher::Regex(r#""blobIds":\["UP"\]"#.into()),
]))
.with_body(
json!({ "methodResponses": [t.parse.clone().unwrap_or(Value::Null)] }).to_string(),
)
.expect(usize::from(t.parse.is_some()))
.create();
let import = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/import".into()))
.with_body(
json!({"methodResponses":[["Email/import",{"accountId":"w",
"created":{"e1":{"id":"Y1","blobId":"UP","threadId":"t","size":10}}},"i"]]})
.to_string(),
)
.expect(t.imports)
.create();
let set = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("Email/set".into()),
Matcher::Regex(r#""keywords/\$seen":true"#.into()),
]))
.with_body(
json!({"methodResponses":[["Email/set",{"accountId":"w","updated":{"X1":null}},"s"]]})
.to_string(),
)
.expect(t.sets)
.create();
let summary = sync::export::run(
common(&archive),
export_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]),
)
.expect("export");
parse.assert();
import.assert();
set.assert();
let _ = std::fs::remove_file(&archive);
summary
.per_type
.iter()
.find(|(t, _)| *t == "Email")
.map(|(_, c)| c.clone())
.expect("email counts")
}
fn parsed_as_b() -> Value {
json!(["Email/parse", {"accountId":"w","parsed":{"UP":{
"messageId":["b@h"],"from":[{"name":null,"email":"a@x"}],"subject":"s","sentAt":null,"to":null}},
"notParsable":[],"notFound":[]}, "p0"])
}
#[test]
fn export_email_the_target_already_holds_under_its_own_key_is_not_imported_again() {
let email = export_against_a_divergent_target_key(DivergentKeyExport {
parse: Some(parsed_as_b()),
local_keywords: "[]",
imports: 0,
sets: 0,
});
assert_eq!((email.created, email.skipped, email.failed), (0, 1, 0));
}
#[test]
fn export_email_matched_by_the_target_parse_gets_its_keywords_synced() {
let email = export_against_a_divergent_target_key(DivergentKeyExport {
parse: Some(parsed_as_b()),
local_keywords: r#"["$seen"]"#,
imports: 0,
sets: 1,
});
assert_eq!((email.created, email.failed), (0, 0));
}
#[test]
fn export_email_is_imported_when_the_target_cannot_parse() {
let email = export_against_a_divergent_target_key(DivergentKeyExport {
parse: Some(json!(["error", {"type":"unknownMethod"}, "p0"])),
local_keywords: "[]",
imports: 1,
sets: 0,
});
assert_eq!((email.created, email.skipped, email.failed), (1, 0, 0));
}
#[test]
fn export_email_parsed_as_a_different_message_is_imported() {
let email = export_against_a_divergent_target_key(DivergentKeyExport {
parse: Some(json!(["Email/parse", {"accountId":"w","parsed":{"UP":{
"messageId":["a@h"],"from":null,"subject":"s","sentAt":null,"to":null}},
"notParsable":[],"notFound":[]}, "p0"])),
local_keywords: "[]",
imports: 1,
sets: 0,
});
assert_eq!((email.created, email.skipped, email.failed), (1, 0, 0));
}