export: batch Email/import, upload in parallel, and show progress
Export wrote one message per request and said nothing while it did, so a large mailbox took hours of silence. Messages now go in batches of up to the target's maxObjectsInSet (at most 50), their blobs uploaded several at once, up to maxConcurrentUpload and no more than --threads; each upload thread reads the archive through its own read-only connection. Every message is still counted on its own: one the target rejects fails alone, a request too large is split, and a method error on the whole call is retried a message at a time. A batch is sent once. If it ends without a clear answer -- a dropped connection, a gateway timeout, a partial failure -- the target is read again, the messages that arrived count as created, and only the rest are imported again, so none is doubled. A progress line (count, rate, time left) is printed every few seconds for mail, contacts and events, and each type ends with a line of what was created, updated, left unchanged and failed.
This commit is contained in:
@@ -487,150 +487,6 @@ fn export_mailbox_already_exists_maps_existing_id() {
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn email_export_sends_one_email_per_import_call() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
for n in 1..=2 {
|
||||
let raw =
|
||||
format!("From: a@x\r\nSubject: m{n}\r\nMessage-ID: <m-{n}@h>\r\n\r\nbody {n}",);
|
||||
let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
|
||||
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')",
|
||||
rusqlite::params![blob],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body(&base))
|
||||
.create();
|
||||
|
||||
let _mq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":["t1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _mq_empty = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Mailbox/query".into()),
|
||||
Matcher::Regex("anchor".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create();
|
||||
let _mg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
|
||||
{"id":"t1","name":"Inbox","role":"inbox","parentId":null,
|
||||
"myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _eq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let _ups = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UP1"}).to_string())
|
||||
.expect(2)
|
||||
.create();
|
||||
|
||||
let single_only = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Email/import".into()),
|
||||
Matcher::Regex("e1".into()),
|
||||
Matcher::Regex("e2".into()),
|
||||
]))
|
||||
.expect(0)
|
||||
.create();
|
||||
|
||||
let imports = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/import",
|
||||
{"accountId":"w","created":{"e":{"id":"x","blobId":"b","threadId":"t","size":10}}},"i"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(2)
|
||||
.create();
|
||||
|
||||
let summary = sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.clone(),
|
||||
threads: 1,
|
||||
dry_run: false,
|
||||
max_retries: 0,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
},
|
||||
ExportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: base.clone(),
|
||||
auth: Auth::Basic {
|
||||
user: "u".into(),
|
||||
password: "p".into(),
|
||||
},
|
||||
account: AccountSelector::Id("w".into()),
|
||||
},
|
||||
objects: None,
|
||||
prune: false,
|
||||
yes: true,
|
||||
},
|
||||
)
|
||||
.expect("export run");
|
||||
|
||||
let email = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Email")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("email counts");
|
||||
assert_eq!(email.created, 2, "both emails imported in per-item rounds");
|
||||
assert_eq!(email.failed, 0, "no per-unit failure");
|
||||
assert!(!summary.any_failed(), "no whole-run failure");
|
||||
|
||||
single_only.assert();
|
||||
imports.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_blob_not_found_reuploads_and_retries() {
|
||||
let mut server = mockito::Server::new();
|
||||
|
||||
Reference in New Issue
Block a user