Export: match a message the target holds under its own key #87
No files matched your search
@@ -13,6 +13,18 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
|
||||
mail-parser 1.0 and rusqlite 0.43. Upstream's new console banner is left
|
||||
out.
|
||||
|
||||
### Fixed
|
||||
- `export` no longer imports a message again on every run when the archive
|
||||
and the target key it differently, as with a message carrying two
|
||||
`Message-ID` headers: the archive keeps both, a JMAP server's `messageId`
|
||||
one. While target emails are left unpaired, a message about to be imported
|
||||
is first parsed by the target (`Email/parse`), and one it already holds is
|
||||
matched instead, with its keywords and folders brought in line like any
|
||||
other match. A target without `Email/parse` gets the message imported as
|
||||
before. The same key settles a batch whose import ended without a clear
|
||||
answer, so one that did arrive is not sent twice. Upstream fixed the same
|
||||
case in v1.0.17; this is our own fix.
|
||||
|
||||
## [27.1-rc.14]
|
||||
|
||||
The fourteenth release candidate of the 27.1 series: POP3, Fastmail, Zoho and
|
||||
|
||||
+166
-9
@@ -20,7 +20,9 @@ use crate::jmap::retry::MethodCallKind;
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
|
||||
use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json};
|
||||
use crate::sync::keys::{
|
||||
EmailIndex, EmailKey, email_index, email_key, email_keys, index_from_json,
|
||||
};
|
||||
use crate::sync::progress::Progress;
|
||||
use crate::sync::{Context, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
@@ -107,6 +109,17 @@ pub fn reconcile(
|
||||
None => creates.push(i),
|
||||
}
|
||||
}
|
||||
let mut paired = vec![false; targets.len()];
|
||||
for t in pairs.iter().flatten() {
|
||||
paired[*t] = true;
|
||||
}
|
||||
let mut unpaired = Unpaired {
|
||||
targets: &targets,
|
||||
keys: &target_keys,
|
||||
free: (0..targets.len()).filter(|t| !paired[*t]).collect(),
|
||||
migrated: &migrated,
|
||||
parse: true,
|
||||
};
|
||||
let mut progress = Progress::new("export: Email", creates.len() as u64, logger);
|
||||
import_units(
|
||||
net,
|
||||
@@ -114,6 +127,8 @@ pub fn reconcile(
|
||||
maps,
|
||||
&units,
|
||||
&creates,
|
||||
&mut unpaired,
|
||||
&mut updates,
|
||||
counts,
|
||||
logger,
|
||||
&mut progress,
|
||||
@@ -170,12 +185,100 @@ fn target_emails(net: &Net) -> Result<(Vec<TargetEmail>, Vec<EmailKey>), JmapErr
|
||||
/// without a clear answer leaves few messages to check.
|
||||
const IMPORT_BATCH_CAP: usize = 50;
|
||||
|
||||
/// One message ready to import: its creation id, its place in `units`, and
|
||||
/// the `Email/import` entry.
|
||||
/// One message ready to import: its creation id, its place in `units`, the
|
||||
/// `Email/import` entry, and the key the target gave it when it parsed the
|
||||
/// uploaded bytes, if it was asked.
|
||||
struct Pending {
|
||||
cid: String,
|
||||
unit: usize,
|
||||
item: Value,
|
||||
parsed: Option<EmailKey>,
|
||||
}
|
||||
|
||||
/// The target emails no archive message paired with by its stored key. A
|
||||
/// message the target already holds can still be among them under another
|
||||
/// key: the archive keys a message by every `Message-ID` header it has, while
|
||||
/// the target's `messageId` reflects one, and the two parsers may differ in
|
||||
/// other ways too. Before such a message is imported, the target parses the
|
||||
/// uploaded bytes and the key it computes is looked up here.
|
||||
struct Unpaired<'t> {
|
||||
targets: &'t [TargetEmail],
|
||||
keys: &'t [EmailKey],
|
||||
free: Vec<usize>,
|
||||
migrated: &'t HashSet<String>,
|
||||
/// False once the target has refused `Email/parse`: not asked again.
|
||||
parse: bool,
|
||||
}
|
||||
|
||||
impl Unpaired<'_> {
|
||||
/// Takes the unpaired target email with `key`, the one of `size` first.
|
||||
fn take(&mut self, key: &EmailKey, size: Option<u64>) -> Option<usize> {
|
||||
let same_key = |t: &usize| &self.keys[*t] == key;
|
||||
let pos = self
|
||||
.free
|
||||
.iter()
|
||||
.position(|t| same_key(t) && size.is_some() && self.targets[*t].size == size)
|
||||
.or_else(|| self.free.iter().position(same_key))?;
|
||||
Some(self.free.remove(pos))
|
||||
}
|
||||
}
|
||||
|
||||
/// The most blobs one `Email/parse` call names, within `maxObjectsInGet`.
|
||||
const PARSE_BATCH_CAP: usize = 50;
|
||||
const PARSE_PROPERTIES: [&str; 5] = ["messageId", "from", "subject", "sentAt", "to"];
|
||||
|
||||
/// The key the target computes for each of `blobs`, by blob id. A blob the
|
||||
/// target could not parse has none. If the target refuses the method, it is
|
||||
/// not asked again this run; any other failure leaves this batch unchecked.
|
||||
/// Either way the messages are then imported as before.
|
||||
fn parsed_keys(
|
||||
net: &Net,
|
||||
blobs: &[&str],
|
||||
unpaired: &mut Unpaired<'_>,
|
||||
logger: &Logger,
|
||||
) -> HashMap<String, EmailKey> {
|
||||
let mut out = HashMap::new();
|
||||
let per_call = (net.limits.max_objects_in_get as usize).clamp(1, PARSE_BATCH_CAP);
|
||||
for chunk in blobs.chunks(per_call) {
|
||||
let mut req = Request::new();
|
||||
req.call(
|
||||
"Email/parse",
|
||||
json!({
|
||||
"accountId": net.account,
|
||||
"blobIds": chunk,
|
||||
"properties": PARSE_PROPERTIES,
|
||||
}),
|
||||
"p",
|
||||
);
|
||||
let answer = req.send(&net.client, &net.api).and_then(|resp| {
|
||||
let mr = resp.first()?.clone();
|
||||
check_method_error(&mr)?;
|
||||
Ok(mr)
|
||||
});
|
||||
let mr = match answer {
|
||||
Ok(mr) => mr,
|
||||
Err(e @ JmapError::Method { .. }) => {
|
||||
logger.warn(&format!(
|
||||
"the target refused Email/parse ({e}); messages whose stored match key differs from the target's are imported without that check"
|
||||
));
|
||||
unpaired.parse = false;
|
||||
return out;
|
||||
}
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"Email/parse failed ({e}); {} message(s) are imported without checking the target for them under another key",
|
||||
chunk.len()
|
||||
));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(parsed) = mr.args.get("parsed").and_then(Value::as_object) {
|
||||
for (blob, email) in parsed {
|
||||
out.insert(blob.clone(), email_key(&server_index(email)));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Writes the messages the target does not have yet. They go in batches of
|
||||
@@ -191,6 +294,8 @@ fn import_units(
|
||||
maps: &Maps,
|
||||
units: &[Unit],
|
||||
creates: &[usize],
|
||||
unpaired: &mut Unpaired<'_>,
|
||||
updates: &mut Vec<(String, Value)>,
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
progress: &mut Progress,
|
||||
@@ -232,6 +337,7 @@ fn import_units(
|
||||
cid,
|
||||
unit: i,
|
||||
item: import_item(blob.0, mids, build_keywords(row), &row.received_at),
|
||||
parsed: None,
|
||||
}),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
@@ -251,6 +357,9 @@ fn import_units(
|
||||
if net.dry_run {
|
||||
counts.created += pending.len() as u64;
|
||||
} else {
|
||||
let pending = match_by_target_parse(
|
||||
net, uploader, maps, units, pending, unpaired, updates, counts, logger,
|
||||
);
|
||||
send_batch(
|
||||
net,
|
||||
uploader,
|
||||
@@ -269,6 +378,50 @@ fn import_units(
|
||||
}
|
||||
}
|
||||
|
||||
/// Asks the target to parse the batch's uploaded messages while unpaired
|
||||
/// target emails remain, and keeps back each one it already holds under the
|
||||
/// key it computes: that email is brought in line like any other match. The
|
||||
/// rest are returned to import, each with the target's key for settling.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn match_by_target_parse(
|
||||
net: &Net,
|
||||
uploader: &Uploader,
|
||||
maps: &Maps,
|
||||
units: &[Unit],
|
||||
pending: Vec<Pending>,
|
||||
unpaired: &mut Unpaired<'_>,
|
||||
updates: &mut Vec<(String, Value)>,
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
) -> Vec<Pending> {
|
||||
if !unpaired.parse || unpaired.free.is_empty() || pending.is_empty() {
|
||||
return pending;
|
||||
}
|
||||
let blobs: Vec<&str> = pending
|
||||
.iter()
|
||||
.filter_map(|p| p.item.get("blobId").and_then(Value::as_str))
|
||||
.collect();
|
||||
let mut keys = parsed_keys(net, &blobs, unpaired, logger);
|
||||
let mut out = Vec::with_capacity(pending.len());
|
||||
for mut p in pending {
|
||||
let blob = p.item.get("blobId").and_then(Value::as_str).unwrap_or("");
|
||||
p.parsed = keys.remove(blob);
|
||||
let row = &units[p.unit].row;
|
||||
let found = p
|
||||
.parsed
|
||||
.as_ref()
|
||||
.and_then(|k| unpaired.take(k, uploader.blob_len(row.blob_local_id)));
|
||||
match found {
|
||||
Some(t) => match email_patch(row, &unpaired.targets[t], maps, unpaired.migrated) {
|
||||
Some(patch) => updates.push((unpaired.targets[t].id.clone(), patch)),
|
||||
None => counts.skipped += 1,
|
||||
},
|
||||
None => out.push(p),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Sends one `Email/import` for `batch` and counts each message's outcome.
|
||||
/// A request too large for the server is split in two; a method error that
|
||||
/// rejects the whole call is retried one message at a time, so the one at
|
||||
@@ -406,12 +559,16 @@ fn settle_unclear(
|
||||
return;
|
||||
}
|
||||
};
|
||||
let keys: Vec<EmailKey> = email_keys(
|
||||
&unclear
|
||||
.iter()
|
||||
.map(|p| index_from_json(&units[p.unit].row.message_match))
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
// The target's own key where it parsed the message: one that arrived is
|
||||
// found under it even when the stored key differs.
|
||||
let keys: Vec<EmailKey> = unclear
|
||||
.iter()
|
||||
.map(|p| {
|
||||
p.parsed
|
||||
.clone()
|
||||
.unwrap_or_else(|| email_key(&index_from_json(&units[p.unit].row.message_match)))
|
||||
})
|
||||
.collect();
|
||||
let sizes: Vec<Option<u64>> = unclear
|
||||
.iter()
|
||||
.map(|p| uploader.blob_len(units[p.unit].row.blob_local_id))
|
||||
|
||||
@@ -454,3 +454,61 @@ fn files_go_into_inbuxa_in_its_own_shape_and_come_back() {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn a_message_with_two_message_ids_is_not_exported_twice() {
|
||||
let server = Inbuxa::start().expect("start inbuxa");
|
||||
let dir = TempDir::new().unwrap();
|
||||
let maildir = dir.path().join("mail");
|
||||
let archive = dir.path().join("archive.sqlite");
|
||||
for sub in ["cur", "new", "tmp"] {
|
||||
fs::create_dir_all(maildir.join(sub)).unwrap();
|
||||
}
|
||||
// The archive keys it by both Message-IDs; inbuxa's messageId has one.
|
||||
fs::write(
|
||||
maildir.join("cur/1:2,S"),
|
||||
"From: [email protected]\r\nTo: [email protected]\r\nSubject: Twice\r\n\
|
||||
Message-ID: <[email protected]>\r\nMessage-ID: <[email protected]>\r\n\
|
||||
Date: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let run = |args: &[&str]| {
|
||||
let out = Command::new(BIN)
|
||||
.args(args)
|
||||
.env("INBUXA_MIGRATE_API_KEY", &server.token)
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
||||
assert!(out.status.success(), "{args:?}: {stderr}");
|
||||
String::from_utf8_lossy(&out.stdout).into_owned()
|
||||
};
|
||||
run(&[
|
||||
"import",
|
||||
"maildir",
|
||||
maildir.to_str().unwrap(),
|
||||
archive.to_str().unwrap(),
|
||||
]);
|
||||
let export = || {
|
||||
run(&[
|
||||
"export",
|
||||
"--url",
|
||||
&server.base_url,
|
||||
"--auth-bearer",
|
||||
"--account-name",
|
||||
ADMIN_EMAIL,
|
||||
"--objects",
|
||||
"mailbox,email",
|
||||
archive.to_str().unwrap(),
|
||||
])
|
||||
};
|
||||
let first = export();
|
||||
assert!(first.contains("Email: created=1"), "{first}");
|
||||
let second = export();
|
||||
assert!(second.contains("Email: created=0"), "{second}");
|
||||
assert_eq!(
|
||||
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
|
||||
[("Twice".to_owned(), 1, vec!["$seen".to_owned()])]
|
||||
);
|
||||
}
|
||||
@@ -6204,3 +6204,210 @@ fn export_file_nodes_in_the_targets_own_shape() {
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
}
|
||||
|
||||
/// A message with two Message-ID headers: the archive keys it by both, while a
|
||||
/// JMAP server's `messageId` (RFC 8621 `header:Message-ID:asMessageIds`)
|
||||
/// reflects one header. The stored key and the target's then differ, and
|
||||
/// only the target's own parse of the bytes can tell that it already has it.
|
||||
const TWO_MESSAGE_IDS: &str =
|
||||
"From: a@x\r\nSubject: s\r\nMessage-ID: <a@h>\r\nMessage-ID: <b@h>\r\n\r\nbody";
|
||||
|
||||
struct DivergentKeyExport {
|
||||
/// The `Email/parse` answer, or `None` when the test expects no call.
|
||||
parse: Option<Value>,
|
||||
/// The local copy's keywords, as stored (a JSON array).
|
||||
local_keywords: &'static str,
|
||||
imports: usize,
|
||||
sets: usize,
|
||||
}
|
||||
|
||||
fn export_against_a_divergent_target_key(t: DivergentKeyExport) -> sync::TypeCounts {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let blob = db::blobs::intern_blob(&conn, TWO_MESSAGE_IDS.as_bytes()).unwrap();
|
||||
let idx =
|
||||
inbuxa_migrate::sync::emailmeta::email_index_from_blob(TWO_MESSAGE_IDS.as_bytes());
|
||||
assert_eq!(
|
||||
idx.mids,
|
||||
["a@h", "b@h"],
|
||||
"the archive keys by every Message-ID"
|
||||
);
|
||||
conn.execute(
|
||||
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match)
|
||||
VALUES (?1,'2020-01-01T00:00:00Z','[1]',?2,?3)",
|
||||
rusqlite::params![
|
||||
blob,
|
||||
t.local_keywords,
|
||||
inbuxa_migrate::sync::keys::index_to_json(&idx)
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_full(&base))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _mbterm = anchor_terminator(&mut server, api, "Mailbox");
|
||||
let _emterm = anchor_terminator(&mut server, api, "Email");
|
||||
let _mq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":["T1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _mg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
|
||||
{"id":"T1","name":"Inbox","role":"inbox","parentId":null,"myRights":{"mayDelete":true}}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _eq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/query",
|
||||
{"accountId":"w","ids":["X1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let _eg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/get",{"accountId":"w","list":[
|
||||
{"id":"X1","messageId":["b@h"],"size":TWO_MESSAGE_IDS.len(),
|
||||
"mailboxIds":{"T1":true},"keywords":{}}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let _up = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UP"}).to_string())
|
||||
.expect_at_most(1)
|
||||
.create();
|
||||
let parse = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Email/parse".into()),
|
||||
Matcher::Regex(r#""blobIds":\["UP"\]"#.into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({ "methodResponses": [t.parse.clone().unwrap_or(Value::Null)] }).to_string(),
|
||||
)
|
||||
.expect(usize::from(t.parse.is_some()))
|
||||
.create();
|
||||
let import = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/import",{"accountId":"w",
|
||||
"created":{"e1":{"id":"Y1","blobId":"UP","threadId":"t","size":10}}},"i"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(t.imports)
|
||||
.create();
|
||||
let set = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Email/set".into()),
|
||||
Matcher::Regex(r#""keywords/\$seen":true"#.into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/set",{"accountId":"w","updated":{"X1":null}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(t.sets)
|
||||
.create();
|
||||
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]),
|
||||
)
|
||||
.expect("export");
|
||||
parse.assert();
|
||||
import.assert();
|
||||
set.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Email")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("email counts")
|
||||
}
|
||||
|
||||
fn parsed_as_b() -> Value {
|
||||
json!(["Email/parse", {"accountId":"w","parsed":{"UP":{
|
||||
"messageId":["b@h"],"from":[{"name":null,"email":"a@x"}],"subject":"s","sentAt":null,"to":null}},
|
||||
"notParsable":[],"notFound":[]}, "p0"])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_the_target_already_holds_under_its_own_key_is_not_imported_again() {
|
||||
let email = export_against_a_divergent_target_key(DivergentKeyExport {
|
||||
parse: Some(parsed_as_b()),
|
||||
local_keywords: "[]",
|
||||
imports: 0,
|
||||
sets: 0,
|
||||
});
|
||||
assert_eq!((email.created, email.skipped, email.failed), (0, 1, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_matched_by_the_target_parse_gets_its_keywords_synced() {
|
||||
let email = export_against_a_divergent_target_key(DivergentKeyExport {
|
||||
parse: Some(parsed_as_b()),
|
||||
local_keywords: r#"["$seen"]"#,
|
||||
imports: 0,
|
||||
sets: 1,
|
||||
});
|
||||
assert_eq!((email.created, email.failed), (0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_is_imported_when_the_target_cannot_parse() {
|
||||
let email = export_against_a_divergent_target_key(DivergentKeyExport {
|
||||
parse: Some(json!(["error", {"type":"unknownMethod"}, "p0"])),
|
||||
local_keywords: "[]",
|
||||
imports: 1,
|
||||
sets: 0,
|
||||
});
|
||||
assert_eq!((email.created, email.skipped, email.failed), (1, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_parsed_as_a_different_message_is_imported() {
|
||||
let email = export_against_a_divergent_target_key(DivergentKeyExport {
|
||||
parse: Some(json!(["Email/parse", {"accountId":"w","parsed":{"UP":{
|
||||
"messageId":["a@h"],"from":null,"subject":"s","sentAt":null,"to":null}},
|
||||
"notParsable":[],"notFound":[]}, "p0"])),
|
||||
local_keywords: "[]",
|
||||
imports: 1,
|
||||
sets: 0,
|
||||
});
|
||||
assert_eq!((email.created, email.skipped, email.failed), (1, 0, 0));
|
||||
}
|
||||
Reference in new issue
Block a user