Merge pull request 'ci: retry release file uploads over HTTP/1.1' (#14) from ci/release-upload-retry into main

inbuxa-server's first GitHub-built release lost 50 MB uploads to
Gitea's release API through Cloudflare (curl 92, HTTP/2 PROTOCOL_ERROR)
on both runs of its binaries job. This job uploads the same way. Uploads
now go over HTTP/1.1, and the asset listing and uploads retry 5 times.

.github/workflows/ci.yml

@@ -150,6 +150,14 @@ jobs:
     env:
       TAG: ${{ github.ref_name }}
     steps:
+      # The release notes are this version's section of CHANGELOG.md, so the
+      # release, and the announcement made from it, say what changed. Checked
+      # out first: a checkout cleans the workspace, and would take dist/ with
+      # it if it ran after the download.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          sparse-checkout: CHANGELOG.md
+          sparse-checkout-cone-mode: false
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           path: dist
@@ -164,24 +172,35 @@ jobs:
           set -euo pipefail
           API="$GITEA_URL/api/v1/repos/$GITHUB_REPOSITORY"
           auth=(-H "Authorization: token $GITEA_TOKEN")
+          # Everything the release carries has to be here before a release is made.
+          for f in inbuxa-migrate-linux-amd64.tar.gz inbuxa-migrate-linux-arm64.tar.gz SHA256SUMS; do
+            [ -s "dist/$f" ] || { echo "::error::dist/$f is missing; not creating a release"; exit 1; }
+          done
+          files="Binaries for linux/amd64 and linux/arm64. Verify with SHA256SUMS."
+          notes="$(awk -v v="${TAG#v}" 'index($0, "## [" v "]") == 1 {f = 1; next}
+                                         f && (/^## / || /^---$/) {exit}
+                                         f' CHANGELOG.md | sed -e '/./,$!d')"
+          if [ -n "$notes" ]; then notes="$notes"$'\n\n'"$files"; else notes="$files"; fi
           # Reuse the Release if the tag already has one (a re-run), else make a
           # draft of our own.
           created=0
           id="$(curl -fsS "${auth[@]}" "$API/releases/tags/$TAG" 2>/dev/null | jq -r '.id // empty' || true)"
           if [ -z "$id" ]; then
             id="$(curl -fsS "${auth[@]}" -H 'Content-Type: application/json' \
-                  -d "$(jq -n --arg t "$TAG" '{tag_name:$t, name:$t, draft:true,
+                  -d "$(jq -n --arg t "$TAG" --arg b "$notes" '{tag_name:$t, name:$t, draft:true, body:$b}')" \
-                        body:"Binaries for linux/amd64 and linux/arm64. Verify with SHA256SUMS."}')" \
                   "$API/releases" | jq -r '.id // empty')"
             [ -n "$id" ] || { echo "::error::could not create the release"; exit 1; }
             created=1
           fi
-          have="$(curl -fsS "${auth[@]}" "$API/releases/$id/assets" | jq -r '.[].name')"
+          # The uploads cross Cloudflare, which dropped 50 MB HTTP/2 uploads
+          # part-way for inbuxa-server v2026.9.30.1 (curl 92). HTTP/1.1, and retry.
+          retry=(--retry 5 --retry-all-errors --retry-delay 15)
+          have="$(curl -fsS "${retry[@]}" "${auth[@]}" "$API/releases/$id/assets" | jq -r '.[].name')"
           for f in dist/*; do
             n="$(basename "$f")"
             if grep -qxF "$n" <<<"$have"; then echo "already attached: $n"; continue; fi
             echo "uploading $n"
-            curl -fsS -o /dev/null "${auth[@]}" --form "attachment=@$f" "$API/releases/$id/assets?name=$n" || {
+            curl -fsS --http1.1 "${retry[@]}" -o /dev/null "${auth[@]}" --form "attachment=@$f" "$API/releases/$id/assets?name=$n" || {
               [ "$created" = 1 ] && curl -sS -o /dev/null "${auth[@]}" -X DELETE "$API/releases/$id"
               exit 1
             }

CHANGELOG.md

@@ -3,7 +3,9 @@
 All notable changes to this project are recorded here. Versions are dates:
 release `v2026.9.30` is version `2026.9.30`.
 
-## [Unreleased] -- 2026.9.30
+## [2026.9.30] -- 2026-09-30
+
+The first release of inbuxa-migrate.
 
 ### Changed
 - Renamed to inbuxa-migrate: the binary, the crate, and the credential
@@ -17,6 +19,34 @@ release `v2026.9.30` is version `2026.9.30`.
 - Released as Linux archives for amd64 and arm64 on the Gitea release page,
   with `SHA256SUMS`. The npm, Homebrew, shell and PowerShell installers and
   the MSI are gone.
+- A second export brings matched items up to date instead of skipping them:
+  flags and folders on mail, changed contacts and events, and edited Sieve
+  scripts.
+- A message filed in several folders is written once, in every one of them,
+  and a resumed export adds any folder it was still missing.
+- Sieve scripts from a Stalwart server keep working on inbuxa: their
+  `vnd.stalwart.*` extension names are renamed to inbuxa's, each rename is
+  logged, and a script that can't be activated fails the export instead of
+  leaving filtering quietly off.
+- `--allow-invalid-certs` covers only the server named with `--url`. Microsoft
+  and Google sign-in are always verified.
+- Export imports mail in batches, uploads in parallel within the target's
+  limits, and prints progress: count, rate and time left.
+- `export --dry-run` predicts what would fail -- messages over the target's
+  upload limit, objects over its request limit, Sieve scripts it would reject
+  -- lists what would be created, updated and skipped, and exits as the real
+  run would.
+
+### Fixed
+- Every connection times out instead of waiting forever on a dropped link,
+  and a timeout is retried like any other transient error.
+- An interrupted IMAP or Maildir import keeps what it wrote, and a message
+  that can't be read is recorded and skipped rather than ending the folder.
+- IMAP and EWS imports hold a bounded amount in memory, in batches capped by
+  size as well as count.
+- The archive is created readable by its owner only, and an existing archive
+  others can read is reported.
+- A dry run no longer writes to the target's address books.
 
 ---
 

docs/usage.md

@@ -83,7 +83,7 @@ inbuxa-migrate import imap \
   [--include <REGEX>...] [--exclude <REGEX>...] [--exclude-special <ROLE>...] \
   [--folder <NAME>...] [--subscribed-only] [--noautomap] \
   [--include-deleted] [--allow-cleartext] [--compress] \
-  [--fetch-batch <N>] [--imap-connections <1..8>] \
+  [--fetch-batch <N>] [--fetch-batch-mib <MIB>] [--imap-connections <1..8>] \
   <ARCHIVE>
 ```
 
@@ -91,6 +91,14 @@ Imports mail, and only mail, from any IMAP server. Folders are chosen with
 `--include` and `--exclude` patterns, or by exact name with `--folder`, but
 not both. `--exclude-special` drops folders by SPECIAL-USE role.
 
+Messages are fetched in chunks of at most `--fetch-batch` messages and
+`--fetch-batch-mib` MiB (32 by default), so a folder of large attachments is
+fetched a little at a time, like any other. A single message larger than the
+cap is fetched on its own. Each chunk is written to the archive as it
+arrives: an interrupted import keeps what it fetched, and the next run picks
+up from there. A message that can't be imported is reported with its folder
+and UID, and the rest of the folder carries on.
+
 ### CalDAV
 
 ```
@@ -170,7 +178,8 @@ inbuxa-migrate import exchange-ews \
   (--auth-basic <USER> [--auth-password <PASS>] \
    | --auth-bearer [TOKEN] [--ews-tenant <T> --ews-client-id <ID> \
                             (--ews-device-code | --ews-client-secret <SECRET>)]) \
-  [--ews-connections <1..8>] [--ews-getitem-batch <N>] [--ews-attachment-batch <N>] \
+  [--ews-connections <1..8>] [--ews-getitem-batch <N>] [--ews-getitem-batch-mib <MIB>] \
+  [--ews-attachment-batch <N>] \
   [--ews-no-syncfolderitems] \
   <ARCHIVE>
 ```
@@ -180,6 +189,12 @@ Imports a mailbox from an on-premises Exchange Server through EWS. Without
 Basic, with a bearer token acquired beforehand, with OAuth's interactive
 device-code flow, or with app-only client credentials.
 
+Items are fetched in GetItem batches of at most `--ews-getitem-batch` items
+and `--ews-getitem-batch-mib` MiB (32 by default), `--ews-connections` at a
+time. The byte cap applies where Exchange reports each item's size, which it
+does on a full listing; items found through an incremental sync are batched
+by count.
+
 For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from October 1, 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on April 1, 2027 it is switched off for every tenant. On-premises Exchange Server is not affected.
 
 ### Microsoft Exchange (Graph)
@@ -229,9 +244,42 @@ what changed at the source in between.
 - **Sieve scripts** are matched by name, and the target's is replaced when
   its content differs from the archive's.
 
+Messages go in batches, as many to an `Email/import` as the target's
+`maxObjectsInSet` allows, up to 50, with their blobs uploaded several at a
+time -- the target's `maxConcurrentUpload`, and no more than `--threads`.
+Each message is still counted on its own: one the target rejects fails
+alone, and the rest of its batch lands. A batch that ends without a clear
+answer -- a dropped connection, a gateway timeout -- is never sent again as
+it was. The target is read first, the messages that arrived are counted as
+created, and only the rest are imported again, so none is ever doubled.
+
+While it runs, export prints a line every few seconds for mail, contacts and
+events: how many of how many, how fast, and about how long is left. Each
+type ends with a line of what was created, updated, left unchanged and
+failed.
+
 `--prune` also deletes what is on the target and not in the archive. It asks
 first; `--yes` answers for it, for scripts. Export speaks JMAP only.
 
+With `--dry-run`, export reads the target and writes nothing to the account.
+It prints the plan in plain words: for each type, how much would be
+created, updated, left unchanged or deleted, and what would fail. It
+catches in advance the failures a real run would hit:
+
+- a message larger than the target's `maxSizeUpload`;
+- a contact, event or other object too large for one request under its
+  `maxSizeRequest`, as happens when a photo is carried inline;
+- a Sieve script the target would reject. Each script that would be written
+  is checked with `SieveScript/validate`, after any `vnd.stalwart.*` names
+  are renamed, so what is checked is what would be uploaded. The check needs
+  the script as a blob, so the dry run uploads Sieve scripts, and only them;
+  a blob that nothing uses is discarded by the server. A target without
+  `SieveScript/validate` gets one warning, and its scripts are not checked.
+
+The plan lists each predicted failure and its reason. When anything would
+fail, the dry run exits 5, as the real run would, so a script can stop
+before it starts.
+
 ## Inspect
 
 ```

src/cli.rs

@@ -277,6 +277,14 @@ struct ImapImportArgs {
     )]
     fetch_batch: usize,
 
+    #[arg(
+        long,
+        value_name = "MIB",
+        default_value_t = 32,
+        help = "Most message bytes per body FETCH chunk, in MiB (one larger message goes alone)"
+    )]
+    fetch_batch_mib: u64,
+
     #[arg(
         long,
         value_name = "N",
@@ -804,6 +812,7 @@ fn resolve_imap_import(args: ImapImportArgs) -> Result<Action, Error> {
             automap: !args.noautomap,
             include_deleted: args.include_deleted,
             fetch_batch: args.fetch_batch,
+            fetch_batch_bytes: args.fetch_batch_mib.max(1).saturating_mul(1024 * 1024),
             imap_connections,
             allow_source_change: args.allow_source_change,
         },
@@ -973,6 +982,14 @@ pub struct ExchangeEwsImportArgs {
     )]
     ews_getitem_batch: usize,
 
+    #[arg(
+        long,
+        value_name = "MIB",
+        default_value_t = 32,
+        help = "Most item bytes per GetItem batch, in MiB (one larger item goes alone)"
+    )]
+    ews_getitem_batch_mib: u64,
+
     #[arg(
         long,
         value_name = "N",
@@ -1022,6 +1039,10 @@ fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result<Action, Er
         auth,
         ews_connections,
         getitem_batch: args.ews_getitem_batch.max(1),
+        getitem_batch_bytes: args
+            .ews_getitem_batch_mib
+            .max(1)
+            .saturating_mul(1024 * 1024),
         attachment_batch: args.ews_attachment_batch.max(1),
         use_syncfolderitems: !args.ews_no_syncfolderitems,
         allow_source_change: args.allow_source_change,

src/exchange_ews/parse.rs

@@ -503,6 +503,8 @@ pub struct FindItemResponse {
 pub struct ItemEntry {
     pub element: String,
     pub id: ItemId,
+    /// `item:Size` in bytes, when the server returned it.
+    pub size: Option<u64>,
 }
 
 pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsError> {
@@ -511,6 +513,7 @@ pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsErro
     let mut buf = Vec::new();
     let mut out = FindItemResponse::default();
     let mut in_root = false;
+    let mut reading_size = false;
     loop {
         buf.clear();
         let (ns, ev) = xml.read_resolved_event_into(&mut buf)?;
@@ -531,7 +534,13 @@ pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsErro
                         out.items.push(ItemEntry {
                             element: local.clone(),
                             id: ItemId::default(),
+                            size: None,
                         });
+                    } else if local.eq_ignore_ascii_case("Size")
+                        && matches!(ev, Event::Start(_))
+                        && !out.items.is_empty()
+                    {
+                        reading_size = true;
                     } else if local.eq_ignore_ascii_case("ItemId")
                         && let Some(last) = out.items.last_mut()
                     {
@@ -539,10 +548,18 @@ pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsErro
                     }
                 }
             }
+            Event::Text(ref t) if reading_size => {
+                if let Some(last) = out.items.last_mut() {
+                    let text: &str = t;
+                    last.size = text.trim().parse().ok();
+                }
+            }
             Event::End(e) => {
                 let local = e.local_name().as_ref().to_owned();
                 if local.eq_ignore_ascii_case("RootFolder") {
                     in_root = false;
+                } else if local.eq_ignore_ascii_case("Size") {
+                    reading_size = false;
                 }
             }
             Event::Eof => break,

src/exchange_ews/xml.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -83,7 +84,12 @@ pub fn find_item_body(
     let mut out = String::with_capacity(512);
     out.push_str("<m:FindItem Traversal=\"");
     out.push_str(traversal.as_str());
-    out.push_str("\"><m:ItemShape><t:BaseShape>IdOnly</t:BaseShape></m:ItemShape>");
+    // item:Size lets GetItem batches be split by bytes as well as by count.
+    out.push_str(
+        "\"><m:ItemShape><t:BaseShape>IdOnly</t:BaseShape>\
+         <t:AdditionalProperties><t:FieldURI FieldURI=\"item:Size\"/></t:AdditionalProperties>\
+         </m:ItemShape>",
+    );
     out.push_str("<m:IndexedPageItemView MaxEntriesReturned=\"");
     out.push_str(&page_size.to_string());
     out.push_str("\" Offset=\"");
@@ -290,6 +296,15 @@ mod tests {
         assert!(body.contains("<t:DistinguishedFolderId Id=\"archiveroot\"/>"));
     }
 
+    #[test]
+    fn find_item_asks_for_item_size() {
+        let folder = FolderId::new("FID", "FCK");
+        let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 50);
+        assert!(body.contains(
+            "<t:BaseShape>IdOnly</t:BaseShape><t:AdditionalProperties><t:FieldURI FieldURI=\"item:Size\"/></t:AdditionalProperties></m:ItemShape>"
+        ));
+    }
+
     #[test]
     fn find_item_paginates_with_offset_and_page_size() {
         let folder = FolderId::new("FID", "FCK");

src/jmap/http.rs

@@ -97,6 +97,10 @@ impl Inner {
 #[derive(Debug, Clone, Copy)]
 enum Kind {
     Api,
+    /// An API call that must not be sent twice: a write the server may
+    /// already have applied when the connection failed. A transport error is
+    /// returned to the caller, which checks the target instead of resending.
+    ApiOnce,
     Upload,
 }
 
@@ -220,6 +224,23 @@ impl HttpClient {
             .map_err(|e| JmapError::Malformed(format!("response is not valid json: {e}")))
     }
 
+    /// As `post_json`, but a transport failure is not retried: the request
+    /// may have reached the server, and sending it again could apply it twice.
+    /// Throttling and `503` answers, which mean it was not processed, are still
+    /// retried.
+    pub fn post_json_once(&self, url: &str, body: &Value) -> Result<Value, JmapError> {
+        let payload = serde_json::to_vec(body)?;
+        let raw = self.execute(
+            Kind::ApiOnce,
+            "POST",
+            url,
+            Some(&payload),
+            Some("application/json"),
+        )?;
+        serde_json::from_slice(&raw)
+            .map_err(|e| JmapError::Malformed(format!("response is not valid json: {e}")))
+    }
+
     pub fn upload(
         &self,
         upload_url: &str,
@@ -298,6 +319,16 @@ impl HttpClient {
                                 body: truncate(&body),
                             });
                         }
+                        // A gateway error may come back after the server
+                        // behind it applied the write: not safe to resend.
+                        StatusOutcome::Retryable
+                            if matches!(kind, Kind::ApiOnce) && matches!(status, 502 | 504) =>
+                        {
+                            return Err(JmapError::HttpStatus {
+                                status,
+                                body: truncate(&body),
+                            });
+                        }
                         StatusOutcome::Retryable => {
                             attempt += 1;
                             self.inner.retries_total.fetch_add(1, Ordering::Relaxed);
@@ -344,6 +375,7 @@ impl HttpClient {
                         }
                     }
                 }
+                Attempt::Transport(err) if matches!(kind, Kind::ApiOnce) => return Err(err),
                 Attempt::Transport(err) => match transport_disposition(&err) {
                     Disposition::Fatal => return Err(err),
                     Disposition::Retryable => {
@@ -659,6 +691,90 @@ mod tests {
     use super::*;
     use crate::net::CertOverride;
 
+    /// A server that reads each request and hangs up without answering: a
+    /// transport failure after the request has been sent. Returns its URL and
+    /// the count of requests it has seen.
+    fn hang_up_server() -> (String, Arc<AtomicU64>) {
+        use std::io::Read;
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let url = format!("http://{}/jmap/api", listener.local_addr().unwrap());
+        let seen = Arc::new(AtomicU64::new(0));
+        let counter = seen.clone();
+        std::thread::spawn(move || {
+            for stream in listener.incoming() {
+                let Ok(mut stream) = stream else { break };
+                let mut buf = [0u8; 4096];
+                let _ = stream.read(&mut buf);
+                counter.fetch_add(1, Ordering::SeqCst);
+            }
+        });
+        (url, seen)
+    }
+
+    fn quick_retries(max_retries: u32) -> RetryPolicy {
+        RetryPolicy {
+            max_retries,
+            base: Duration::from_millis(1),
+            cap: Duration::from_millis(2),
+        }
+    }
+
+    #[test]
+    fn a_write_sent_once_is_not_resent_after_a_transport_failure() {
+        let (url, seen) = hang_up_server();
+        let client = HttpClient::new(
+            Auth::Bearer { token: "t".into() },
+            quick_retries(2),
+            CertOverride::none(),
+        );
+        let err = client
+            .post_json_once(&url, &serde_json::json!({}))
+            .unwrap_err();
+        assert!(matches!(err, JmapError::Transport(_)), "{err}");
+        assert_eq!(seen.load(Ordering::SeqCst), 1, "sent exactly once");
+
+        let (url, seen) = hang_up_server();
+        let _ = client.post_json(&url, &serde_json::json!({}));
+        assert_eq!(
+            seen.load(Ordering::SeqCst),
+            3,
+            "an ordinary call retries twice"
+        );
+    }
+
+    #[test]
+    fn a_write_sent_once_is_not_resent_after_a_gateway_timeout_but_is_after_503() {
+        let mut server = mockito::Server::new();
+        let url = format!("{}/jmap/api", server.url());
+        let client = HttpClient::new(
+            Auth::Bearer { token: "t".into() },
+            quick_retries(2),
+            CertOverride::none(),
+        );
+        let gateway = server
+            .mock("POST", "/jmap/api")
+            .with_status(504)
+            .expect(1)
+            .create();
+        let err = client
+            .post_json_once(&url, &serde_json::json!({}))
+            .unwrap_err();
+        assert!(
+            matches!(err, JmapError::HttpStatus { status: 504, .. }),
+            "{err}"
+        );
+        gateway.assert();
+        gateway.remove();
+
+        let busy = server
+            .mock("POST", "/jmap/api")
+            .with_status(503)
+            .expect(3)
+            .create();
+        let _ = client.post_json_once(&url, &serde_json::json!({}));
+        busy.assert();
+    }
+
     #[test]
     fn invalid_certificates_are_accepted_only_for_the_named_host() {
         let client = HttpClient::new(

src/jmap/request.rs

@@ -131,6 +131,14 @@ impl Request {
         let value = client.post_json(api_url, &self.envelope()?)?;
         Response::parse(value)
     }
+
+    /// As `send`, for a write that must not be applied twice: a transport
+    /// failure comes back as an error instead of being resent (see
+    /// `HttpClient::post_json_once`).
+    pub fn send_once(&self, client: &HttpClient, api_url: &str) -> Result<Response, JmapError> {
+        let value = client.post_json_once(api_url, &self.envelope()?)?;
+        Response::parse(value)
+    }
 }
 
 #[derive(Debug)]

src/main.rs

@@ -31,6 +31,7 @@ fn run() -> i32 {
         Err(err) => return fail(&err),
     };
 
+    let mut quiet_report = false;
     let (outcome, logger) = match action {
         Action::Import(common, config) => {
             let logger = common.logger;
@@ -75,6 +76,8 @@ fn run() -> i32 {
         }
         Action::Export(common, config) => {
             let logger = common.logger;
+            // A dry run prints its own plan; the counts below would repeat it.
+            quiet_report = common.dry_run;
             (
                 RunOutcome::from_result(sync::export::run(common, config)),
                 logger,
@@ -88,7 +91,9 @@ fn run() -> i32 {
         }
     };
 
+    if !quiet_report {
         report(&outcome.summary);
+    }
     match outcome.error {
         Some(err) => fail(&err),
         None => {

src/sync/batch.rs

@@ -0,0 +1,92 @@
+/*
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+//! Fetch batches bounded by bytes as well as by count. A batch of a few
+//! hundred messages is small for ordinary mail and many gigabytes for a
+//! mailbox of large attachments; capping the bytes too keeps what a batch
+//! holds in memory about the same whatever the mail is like.
+
+/// The default byte cap for one fetch batch.
+pub const DEFAULT_BATCH_BYTES: u64 = 32 * 1024 * 1024;
+
+/// Splits `items` into contiguous batches of at most `max_count` items and
+/// at most `max_bytes` bytes, as reported by `size`. An item whose size is
+/// unknown counts as 0 bytes, so without sizes this is batching by count. An
+/// item larger than `max_bytes` goes in a batch of its own: every batch has
+/// at least one item.
+pub fn by_count_and_bytes<T>(
+    items: &[T],
+    size: impl Fn(&T) -> u64,
+    max_count: usize,
+    max_bytes: u64,
+) -> Vec<&[T]> {
+    let max_count = max_count.max(1);
+    let max_bytes = max_bytes.max(1);
+    let mut out = Vec::new();
+    let mut start = 0usize;
+    let mut bytes = 0u64;
+    for (i, item) in items.iter().enumerate() {
+        let s = size(item);
+        let count = i - start;
+        if count > 0 && (count >= max_count || bytes.saturating_add(s) > max_bytes) {
+            out.push(&items[start..i]);
+            start = i;
+            bytes = 0;
+        }
+        bytes = bytes.saturating_add(s);
+    }
+    if start < items.len() {
+        out.push(&items[start..]);
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn batches(sizes: &[u64], max_count: usize, max_bytes: u64) -> Vec<Vec<u64>> {
+        by_count_and_bytes(sizes, |s| *s, max_count, max_bytes)
+            .into_iter()
+            .map(|b| b.to_vec())
+            .collect()
+    }
+
+    #[test]
+    fn respects_the_byte_cap() {
+        assert_eq!(
+            batches(&[10, 10, 10, 10, 10], 100, 25),
+            vec![vec![10, 10], vec![10, 10], vec![10]]
+        );
+    }
+
+    #[test]
+    fn respects_the_count_cap() {
+        assert_eq!(
+            batches(&[1, 1, 1, 1, 1], 2, 1000),
+            vec![vec![1, 1], vec![1, 1], vec![1]]
+        );
+    }
+
+    #[test]
+    fn an_item_over_the_cap_goes_alone() {
+        assert_eq!(
+            batches(&[5, 500, 5, 5], 100, 20),
+            vec![vec![5], vec![500], vec![5, 5]]
+        );
+        assert_eq!(batches(&[500], 100, 20), vec![vec![500]]);
+    }
+
+    #[test]
+    fn unknown_sizes_batch_by_count() {
+        assert_eq!(batches(&[0, 0, 0], 2, 1), vec![vec![0, 0], vec![0]]);
+    }
+
+    #[test]
+    fn nothing_in_nothing_out() {
+        assert!(batches(&[], 10, 10).is_empty());
+    }
+}

src/sync/export.rs

@@ -7,6 +7,9 @@
 
 use std::collections::{HashMap, HashSet};
 use std::io::{IsTerminal, Write};
+use std::path::PathBuf;
+use std::sync::Mutex;
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use rusqlite::Connection;
 use serde_json::{Map, Value, json};
@@ -88,8 +91,9 @@ impl<'a> Uploader<'a> {
             return Ok(id.clone());
         }
         let id = if self.net.dry_run {
-            let _exists = db::blobs::blob_bytes(self.conn, local_id)?
+            let len = db::blobs::blob_len(self.conn, local_id)?
                 .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?;
+            self.net.check_upload_size(len)?;
             JmapId(format!("dryrun-blob-{local_id}"))
         } else {
             let bytes = db::blobs::blob_bytes(self.conn, local_id)?
@@ -120,6 +124,7 @@ impl<'a> Uploader<'a> {
             return Ok(id.clone());
         }
         let id = if self.net.dry_run {
+            self.net.check_upload_size(bytes.len() as u64)?;
             JmapId(format!("dryrun-blob-{local_id}"))
         } else {
             blobxfer::upload_bytes(
@@ -134,6 +139,77 @@ impl<'a> Uploader<'a> {
         Ok(id)
     }
 
+    /// Uploads several stored blobs at once, on up to `Net::upload_workers`
+    /// threads, and returns each one's result in the order given. Each thread
+    /// reads its blobs through its own read-only connection to the archive, so
+    /// at most one blob per thread is held in memory. With one worker, or in a
+    /// dry run, it is `upload_with` in a loop.
+    fn upload_many(
+        &mut self,
+        local_ids: &[i64],
+        content_type: &str,
+    ) -> Vec<Result<JmapId, JmapError>> {
+        if self.net.dry_run || self.net.upload_workers <= 1 {
+            return local_ids
+                .iter()
+                .map(|id| self.upload_with(*id, content_type))
+                .collect();
+        }
+        self.touched.extend_from_slice(local_ids);
+        let mut todo: Vec<i64> = Vec::new();
+        for id in local_ids {
+            if !self.cache.contains_key(id) && !todo.contains(id) {
+                todo.push(*id);
+            }
+        }
+        let net = self.net;
+        let results = run_bounded(
+            &todo,
+            net.upload_workers,
+            || {
+                rusqlite::Connection::open_with_flags(
+                    &net.archive,
+                    rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
+                )
+            },
+            |conn, local_id| {
+                let conn = conn
+                    .as_ref()
+                    .map_err(|e| JmapError::malformed(format!("archive not readable: {e}")))?;
+                let bytes = db::blobs::blob_bytes(conn, *local_id)?.ok_or_else(|| {
+                    JmapError::malformed(format!("blob local id {local_id} missing"))
+                })?;
+                blobxfer::upload_bytes(
+                    &net.client,
+                    &net.session,
+                    &net.account,
+                    content_type,
+                    &bytes,
+                )
+            },
+        );
+        let mut failed: HashMap<i64, JmapError> = HashMap::new();
+        for (local_id, result) in todo.into_iter().zip(results) {
+            match result {
+                Ok(id) => {
+                    self.cache.insert(local_id, id);
+                }
+                Err(e) => {
+                    failed.insert(local_id, e);
+                }
+            }
+        }
+        local_ids
+            .iter()
+            .map(|id| match self.cache.get(id) {
+                Some(blob) => Ok(blob.clone()),
+                None => Err(failed.get(id).map(clone_error).unwrap_or_else(|| {
+                    JmapError::malformed(format!("blob local id {id} not uploaded"))
+                })),
+            })
+            .collect()
+    }
+
     fn invalidate(&mut self, local_id: i64) {
         self.cache.remove(&local_id);
     }
@@ -147,6 +223,58 @@ impl<'a> Uploader<'a> {
     }
 }
 
+/// A copy of an upload error for each archive row that shares the blob. The
+/// errors that carry meaning for the caller -- the size limits -- keep their
+/// kind; the rest keep their message.
+fn clone_error(e: &JmapError) -> JmapError {
+    match e {
+        JmapError::RequestTooLarge => JmapError::RequestTooLarge,
+        JmapError::SingleObjectTooLarge(m) => JmapError::SingleObjectTooLarge(m.clone()),
+        other => JmapError::Transport(other.to_string()),
+    }
+}
+
+/// Runs `f` over `jobs` on at most `workers` threads and returns the results
+/// in job order. Each thread builds its own state once with `init`, such as a
+/// connection of its own to the archive.
+fn run_bounded<J, S, R>(
+    jobs: &[J],
+    workers: usize,
+    init: impl Fn() -> S + Sync,
+    f: impl Fn(&mut S, &J) -> R + Sync,
+) -> Vec<R>
+where
+    J: Sync,
+    R: Send,
+{
+    let workers = workers.clamp(1, jobs.len().max(1));
+    if workers == 1 {
+        let mut state = init();
+        return jobs.iter().map(|j| f(&mut state, j)).collect();
+    }
+    let next = AtomicUsize::new(0);
+    let slots: Mutex<Vec<Option<R>>> = Mutex::new((0..jobs.len()).map(|_| None).collect());
+    std::thread::scope(|scope| {
+        for _ in 0..workers {
+            scope.spawn(|| {
+                let mut state = init();
+                loop {
+                    let i = next.fetch_add(1, Ordering::SeqCst);
+                    let Some(job) = jobs.get(i) else { break };
+                    let r = f(&mut state, job);
+                    slots.lock().expect("result slots")[i] = Some(r);
+                }
+            });
+        }
+    });
+    slots
+        .into_inner()
+        .expect("result slots")
+        .into_iter()
+        .map(|r| r.expect("every job ran"))
+        .collect()
+}
+
 impl BlobBytes for Uploader<'_> {
     fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
         db::blobs::blob_bytes(self.conn, local_id)?
@@ -162,6 +290,40 @@ struct Net {
     limits: Limits,
     session: Session,
     dry_run: bool,
+    /// The archive's path, for the upload threads' own connections.
+    archive: PathBuf,
+    /// Blobs uploaded at once: the server's `maxConcurrentUpload`, and no
+    /// more than `--threads`.
+    upload_workers: usize,
+    /// In a dry run, what would fail and why, for the plan.
+    would_fail: std::sync::Arc<Mutex<Vec<String>>>,
+}
+
+impl Net {
+    /// In a dry run, a blob of `len` bytes that the target would refuse:
+    /// over its `maxSizeUpload`.
+    fn check_upload_size(&self, len: u64) -> Result<(), JmapError> {
+        let cap = self.limits.max_size_upload;
+        if cap > 0 && len > cap {
+            return Err(JmapError::SingleObjectTooLarge(format!(
+                "{} is larger than the target accepts ({} maxSizeUpload)",
+                crate::inspect::format_bytes(len),
+                crate::inspect::format_bytes(cap)
+            )));
+        }
+        Ok(())
+    }
+
+    /// Notes, in a dry run, that `what` would fail and why. A real run
+    /// reports failures as they happen and keeps no list.
+    fn would_fail(&self, what: impl Into<String>) {
+        if self.dry_run {
+            self.would_fail
+                .lock()
+                .expect("would-fail list")
+                .push(what.into());
+        }
+    }
 }
 
 fn has_rows(conn: &Connection, ty: ObjectType) -> bool {
@@ -185,6 +347,11 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
         limits: connected.limits,
         session: connected.session.clone(),
         dry_run: ctx.dry_run(),
+        archive: ctx.common.archive.clone(),
+        upload_workers: (connected.limits.max_concurrent_upload as usize)
+            .min(ctx.common.threads)
+            .max(1),
+        would_fail: Default::default(),
     };
 
     let work = work_list(&ctx.conn, &config, &connected, &logger);
@@ -198,6 +365,7 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
         if logger.enabled(LEVEL_DEFAULT) {
             eprintln!("export: {} ...", ty.jmap_name());
         }
+        let started = std::time::Instant::now();
         let mut counts = TypeCounts::default();
         let res = reconcile_type(
             &ctx,
@@ -217,6 +385,16 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
                 Plan::default()
             }
         };
+        if logger.enabled(LEVEL_DEFAULT) && !ctx.dry_run() {
+            eprintln!(
+                "{}",
+                crate::sync::progress::done_line(
+                    &format!("export: {}", ty.jmap_name()),
+                    &counts,
+                    started.elapsed()
+                )
+            );
+        }
         plans.insert(*ty, plan);
         counts_per_type.insert(*ty, counts);
     }
@@ -240,8 +418,9 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
     }
 
     if ctx.dry_run() {
-        print_dry_run(&dry_rows, config.prune);
+        let would_fail = net.would_fail.lock().expect("would-fail list").clone();
-        return Ok(Summary::default());
+        print_plan(&summary, &dry_rows, &would_fail, config.prune);
+        return Ok(summary);
     }
     summary.retries_observed = ctx.client.retries_observed();
     summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
@@ -447,21 +626,69 @@ fn sample(ids: &[String]) -> String {
     ids[..n].join(", ")
 }
 
-fn print_dry_run(rows: &[(&'static str, u64, u64, u64)], prune: bool) {
+/// The dry run's report, in plain words: per type, what would be created,
+/// updated, left as it is and would fail; then why each failure would happen.
+fn print_plan(
+    summary: &Summary,
+    dry_rows: &[(&'static str, u64, u64, u64)],
+    would_fail: &[String],
+    prune: bool,
+) {
+    print!("{}", plan_text(summary, dry_rows, would_fail, prune));
+}
+
+fn plan_text(
+    summary: &Summary,
+    dry_rows: &[(&'static str, u64, u64, u64)],
+    would_fail: &[String],
+    prune: bool,
+) -> String {
+    use crate::sync::progress::thousands;
+    let mut out = String::from("Dry run: nothing was written to the target. The plan:\n");
+    for (ty, c) in &summary.per_type {
+        let mut parts: Vec<String> = Vec::new();
+        if c.created > 0 {
+            parts.push(format!("{} to create", thousands(c.created)));
+        }
+        if c.updated > 0 {
+            parts.push(format!("{} to update", thousands(c.updated)));
+        }
+        if c.skipped > 0 {
+            parts.push(format!("{} unchanged", thousands(c.skipped)));
+        }
+        if c.failed > 0 {
+            parts.push(format!("{} would fail", thousands(c.failed)));
+        }
         if prune {
-        println!(
+            let gone = dry_rows
-            "{:<22} {:>10} {:>10} {:>12}",
+                .iter()
-            "TYPE", "CREATE", "MATCHED", "WOULD-DESTROY"
+                .find(|(t, ..)| t == ty)
-        );
+                .map(|(.., d)| *d)
-        for (ty, c, m, d) in rows {
+                .unwrap_or(0);
-            println!("{ty:<22} {c:>10} {m:>10} {d:>12}");
+            if gone > 0 {
-        }
+                parts.push(format!("{} to delete (--prune)", thousands(gone)));
-    } else {
-        println!("{:<22} {:>10} {:>10}", "TYPE", "CREATE", "MATCHED");
-        for (ty, c, m, _) in rows {
-            println!("{ty:<22} {c:>10} {m:>10}");
             }
         }
+        if parts.is_empty() {
+            parts.push("nothing to do".to_owned());
+        }
+        out.push_str(&format!("  {ty:<20} {}\n", parts.join(", ")));
+    }
+    let failed: u64 = summary.per_type.iter().map(|(_, c)| c.failed).sum();
+    if failed > 0 {
+        out.push_str("Would fail:\n");
+        for line in would_fail {
+            out.push_str(&format!("  {line}\n"));
+        }
+        let unexplained = failed.saturating_sub(would_fail.len() as u64);
+        if unexplained > 0 {
+            out.push_str(&format!(
+                "  {} more; the warnings above say why\n",
+                thousands(unexplained)
+            ));
+        }
+    }
+    out
 }
 
 mod tree;
@@ -522,7 +749,7 @@ mod common {
         creates: Vec<(String, Value)>,
     ) -> Result<crate::jmap::request::SetOutcome, JmapError> {
         if net.dry_run {
-            return Ok(synthesize_dry_run_outcome(ty, &creates));
+            return Ok(synthesize_dry_run_outcome(net, ty, &creates));
         }
         let mut map = Map::new();
         for (cid, obj) in creates {
@@ -621,12 +848,35 @@ mod common {
         create_batch(net, ty, vec![(cid.to_owned(), wire)]).map_err(Error::from)
     }
 
+    /// Room left in a request for everything but the object itself: the
+    /// envelope, the method name and the arguments around it.
+    const REQUEST_OVERHEAD: u64 = 512;
+
+    /// What a dry run predicts for `creates`: each one created, except an
+    /// object too big to fit in one request under the target's
+    /// `maxSizeRequest`, which a real run could not send either.
     fn synthesize_dry_run_outcome(
+        net: &Net,
         ty: ObjectType,
         creates: &[(String, Value)],
     ) -> crate::jmap::request::SetOutcome {
         let mut outcome = crate::jmap::request::SetOutcome::default();
-        for (cid, _) in creates {
+        let cap = net.limits.max_size_request;
+        for (cid, obj) in creates {
+            let size = serde_json::to_vec(obj).map(|v| v.len() as u64).unwrap_or(0);
+            if cap > 0 && size + REQUEST_OVERHEAD > cap {
+                let why = format!(
+                    "{} is larger than one request to the target may be ({} maxSizeRequest)",
+                    crate::inspect::format_bytes(size),
+                    crate::inspect::format_bytes(cap)
+                );
+                net.would_fail(format!("{} {cid}: {why}", ty.jmap_name()));
+                outcome.not_created.push((
+                    cid.clone(),
+                    serde_json::json!({ "type": "tooLarge", "description": why }),
+                ));
+                continue;
+            }
             let synthetic = serde_json::json!({
                 "id": format!("dryrun-{}-{cid}", ty.jmap_name())
             });
@@ -635,3 +885,120 @@ mod common {
         outcome
     }
 }
+
+#[cfg(test)]
+mod pool_tests {
+    use super::run_bounded;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Duration;
+
+    #[test]
+    fn never_more_workers_at_once_than_the_cap_and_results_keep_job_order() {
+        let jobs: Vec<usize> = (0..24).collect();
+        let running = AtomicUsize::new(0);
+        let most = AtomicUsize::new(0);
+        let inits = AtomicUsize::new(0);
+        let out = run_bounded(
+            &jobs,
+            3,
+            || inits.fetch_add(1, Ordering::SeqCst),
+            |_, j| {
+                let now = running.fetch_add(1, Ordering::SeqCst) + 1;
+                most.fetch_max(now, Ordering::SeqCst);
+                std::thread::sleep(Duration::from_millis(5));
+                running.fetch_sub(1, Ordering::SeqCst);
+                j * 2
+            },
+        );
+        assert_eq!(out, jobs.iter().map(|j| j * 2).collect::<Vec<_>>());
+        let most = most.load(Ordering::SeqCst);
+        assert!(most <= 3, "{most} ran at once");
+        assert!(most > 1, "work ran in parallel");
+        assert_eq!(
+            inits.load(Ordering::SeqCst),
+            3,
+            "state built once per worker"
+        );
+    }
+
+    #[test]
+    fn one_worker_or_one_job_runs_in_place() {
+        let out = run_bounded(&[1, 2, 3], 1, || (), |_, j| j + 1);
+        assert_eq!(out, vec![2, 3, 4]);
+        let out = run_bounded(&[7], 8, || (), |_, j| j + 1);
+        assert_eq!(out, vec![8]);
+        let out: Vec<i32> = run_bounded(&[], 4, || (), |_, j: &i32| *j);
+        assert!(out.is_empty());
+    }
+}
+
+#[cfg(test)]
+mod plan_tests {
+    use super::plan_text;
+    use crate::sync::{Summary, TypeCounts};
+
+    fn summary(rows: &[(&'static str, u64, u64, u64, u64)]) -> Summary {
+        Summary {
+            per_type: rows
+                .iter()
+                .map(|(t, created, updated, skipped, failed)| {
+                    (
+                        *t,
+                        TypeCounts {
+                            created: *created,
+                            updated: *updated,
+                            skipped: *skipped,
+                            failed: *failed,
+                            ..Default::default()
+                        },
+                    )
+                })
+                .collect(),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn the_plan_reads_in_plain_words_and_says_why_things_would_fail() {
+        let s = summary(&[
+            ("Mailbox", 0, 0, 12, 0),
+            ("Email", 1200, 40, 5000, 2),
+            ("SieveScript", 0, 0, 0, 0),
+        ]);
+        let text = plan_text(
+            &s,
+            &[],
+            &["Email e7 (message-id <a@b>): 61 MB is larger than the target accepts".to_owned()],
+            false,
+        );
+        assert!(
+            text.starts_with("Dry run: nothing was written to the target."),
+            "{text}"
+        );
+        assert!(text.contains("Mailbox              12 unchanged"), "{text}");
+        assert!(
+            text.contains(
+                "Email                1,200 to create, 40 to update, 5,000 unchanged, 2 would fail"
+            ),
+            "{text}"
+        );
+        assert!(
+            text.contains("SieveScript          nothing to do"),
+            "{text}"
+        );
+        assert!(text.contains("Would fail:\n  Email e7"), "{text}");
+        assert!(
+            text.contains("1 more; the warnings above say why"),
+            "{text}"
+        );
+    }
+
+    #[test]
+    fn prune_counts_appear_only_with_prune() {
+        let s = summary(&[("ContactCard", 0, 0, 3, 0)]);
+        let rows = [("ContactCard", 0, 3, 4)];
+        assert!(plan_text(&s, &rows, &[], true).contains("3 unchanged, 4 to delete (--prune)"));
+        assert!(!plan_text(&s, &rows, &[], false).contains("delete"));
+        assert!(!plan_text(&s, &rows, &[], false).contains("Would fail"));
+    }
+}

src/sync/export/email.rs

@@ -21,6 +21,7 @@ use crate::jmap::wire::JmapId;
 use crate::logging::Logger;
 use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
 use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json};
+use crate::sync::progress::Progress;
 use crate::sync::{Context, TypeCounts};
 use crate::types::ObjectType;
 
@@ -61,45 +62,7 @@ pub fn reconcile(
     logger: &Logger,
 ) -> Result<Plan, Error> {
     let ty = ObjectType::Email;
-
+    let (targets, target_keys) = target_emails(net).map_err(Error::from)?;
-    let target_min = target_query_get(
-        net,
-        ty,
-        Some(&["messageId", "size", "mailboxIds", "keywords"]),
-    )
-    .map_err(Error::from)?;
-    let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
-
-    let fallback_ids: Vec<JmapId> = target_min
-        .iter()
-        .zip(indices.iter())
-        .filter(|(_, i)| i.mids.is_empty())
-        .filter_map(|(v, _)| jid(v).map(JmapId))
-        .collect();
-    if !fallback_ids.is_empty() {
-        let got = get_objects::<Value>(
-            &net.client,
-            &net.api,
-            &net.account,
-            ty.jmap_name(),
-            &fallback_ids,
-            Some(&["messageId", "from", "subject", "sentAt", "to"]),
-            &net.limits,
-        )
-        .map_err(Error::from)?;
-        let by_id: HashMap<String, &Value> = got
-            .list
-            .iter()
-            .filter_map(|v| jid(v).map(|i| (i, v)))
-            .collect();
-        for (v, slot) in target_min.iter().zip(indices.iter_mut()) {
-            if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) {
-                *slot = server_index(full);
-            }
-        }
-    }
-    let targets: Vec<TargetEmail> = target_min.iter().map(TargetEmail::from_value).collect();
-    let target_keys = email_keys(&indices);
 
     let mut local: Vec<(i64, EmailRow)> = {
         let mut stmt = ctx
@@ -134,27 +97,343 @@ pub fn reconcile(
 
     let migrated = maps.targets_of(ObjectType::Mailbox);
     let mut updates: Vec<(String, Value)> = Vec::new();
+    let mut creates: Vec<usize> = Vec::new();
     for (i, unit) in units.iter().enumerate() {
         match pairs[i] {
             Some(t) => match email_patch(&unit.row, &targets[t], maps, &migrated) {
                 Some(patch) => updates.push((targets[t].id.clone(), patch)),
                 None => counts.skipped += 1,
             },
-            None => export_one(
+            None => creates.push(i),
+        }
+    }
+    let mut progress = Progress::new("export: Email", creates.len() as u64, logger);
+    import_units(
         net,
         &mut uploader,
         maps,
+        &units,
+        &creates,
+        counts,
+        logger,
+        &mut progress,
+    );
+    update_batch(net, ty, updates, counts, logger);
+
+    Ok(Plan::default())
+}
+
+/// The emails already on the target, and the key each one matches by.
+fn target_emails(net: &Net) -> Result<(Vec<TargetEmail>, Vec<EmailKey>), JmapError> {
+    let ty = ObjectType::Email;
+    let target_min = target_query_get(
+        net,
+        ty,
+        Some(&["messageId", "size", "mailboxIds", "keywords"]),
+    )?;
+    let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
+
+    let fallback_ids: Vec<JmapId> = target_min
+        .iter()
+        .zip(indices.iter())
+        .filter(|(_, i)| i.mids.is_empty())
+        .filter_map(|(v, _)| jid(v).map(JmapId))
+        .collect();
+    if !fallback_ids.is_empty() {
+        let got = get_objects::<Value>(
+            &net.client,
+            &net.api,
+            &net.account,
+            ty.jmap_name(),
+            &fallback_ids,
+            Some(&["messageId", "from", "subject", "sentAt", "to"]),
+            &net.limits,
+        )?;
+        let by_id: HashMap<String, &Value> = got
+            .list
+            .iter()
+            .filter_map(|v| jid(v).map(|i| (i, v)))
+            .collect();
+        for (v, slot) in target_min.iter().zip(indices.iter_mut()) {
+            if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) {
+                *slot = server_index(full);
+            }
+        }
+    }
+    let targets: Vec<TargetEmail> = target_min.iter().map(TargetEmail::from_value).collect();
+    let keys = email_keys(&indices);
+    Ok((targets, keys))
+}
+
+/// The most emails one `Email/import` carries: the server's
+/// `maxObjectsInSet`, but no more than this, so that a request that fails
+/// without a clear answer leaves few messages to check.
+const IMPORT_BATCH_CAP: usize = 50;
+
+/// One message ready to import: its creation id, its place in `units`, and
+/// the `Email/import` entry.
+struct Pending {
+    cid: String,
+    unit: usize,
+    item: Value,
+}
+
+/// Writes the messages the target does not have yet. They go in batches of
+/// up to `maxObjectsInSet` (capped by `IMPORT_BATCH_CAP`), their blobs
+/// uploaded at once up to `maxConcurrentUpload`. Each message is still
+/// counted on its own: one rejected in a batch fails alone. A batch that
+/// fails without a clear answer is never resent blindly -- the target is
+/// checked first, and only what did not arrive is imported again.
+#[allow(clippy::too_many_arguments)]
+fn import_units(
+    net: &Net,
+    uploader: &mut Uploader,
+    maps: &Maps,
+    units: &[Unit],
+    creates: &[usize],
+    counts: &mut TypeCounts,
+    logger: &Logger,
+    progress: &mut Progress,
+) {
+    let batch = (net.limits.max_objects_in_set as usize).clamp(1, IMPORT_BATCH_CAP);
+    let mut unclear: Vec<Pending> = Vec::new();
+    for chunk in creates.chunks(batch) {
+        let mut ready: Vec<(usize, Map<String, Value>)> = Vec::new();
+        for &i in chunk {
+            let row = &units[i].row;
+            match build_mailbox_ids(row, maps) {
+                Some(mids) => ready.push((i, mids)),
+                None => {
+                    logger.warn(&format!(
+                        "Email/import e{} ({}) skipped: mailbox not on target",
+                        units[i].local_id,
+                        blob_hint(uploader, row)
+                    ));
+                    net.would_fail(format!(
+                        "Email e{} ({}): its folder is not on the target",
+                        units[i].local_id,
+                        blob_hint(uploader, row)
+                    ));
+                    counts.failed += 1;
+                }
+            }
+        }
+        let blobs: Vec<i64> = ready
+            .iter()
+            .map(|(i, _)| units[*i].row.blob_local_id)
+            .collect();
+        let uploaded = uploader.upload_many(&blobs, "message/rfc822");
+        let mut pending: Vec<Pending> = Vec::new();
+        for ((i, mids), result) in ready.into_iter().zip(uploaded) {
+            let row = &units[i].row;
+            let cid = format!("e{}", units[i].local_id);
+            match result {
+                Ok(blob) => pending.push(Pending {
+                    cid,
+                    unit: i,
+                    item: import_item(blob.0, mids, build_keywords(row), &row.received_at),
+                }),
+                Err(e) => {
+                    logger.warn(&format!(
+                        "Email/import {cid} ({}) blob upload failed: {e}{}",
+                        blob_hint(uploader, row),
+                        size_note(&e)
+                    ));
+                    net.would_fail(format!(
+                        "Email {cid} ({}): {}",
+                        blob_hint(uploader, row),
+                        plain_reason(&e)
+                    ));
+                    counts.failed += 1;
+                }
+            }
+        }
+        if net.dry_run {
+            counts.created += pending.len() as u64;
+        } else {
+            send_batch(
+                net,
+                uploader,
+                maps,
+                units,
+                pending,
+                counts,
+                logger,
+                &mut unclear,
+            );
+        }
+        progress.add(chunk.len() as u64);
+    }
+    if !unclear.is_empty() {
+        settle_unclear(net, uploader, maps, units, unclear, counts, logger);
+    }
+}
+
+/// Sends one `Email/import` for `batch` and counts each message's outcome.
+/// A request too large for the server is split in two; a method error that
+/// rejects the whole call is retried one message at a time, so the one at
+/// fault fails alone. Anything that leaves it unclear whether the server
+/// applied the call goes to `unclear`.
+#[allow(clippy::too_many_arguments)]
+fn send_batch(
+    net: &Net,
+    uploader: &mut Uploader,
+    maps: &Maps,
+    units: &[Unit],
+    batch: Vec<Pending>,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+    unclear: &mut Vec<Pending>,
+) {
+    if batch.is_empty() {
+        return;
+    }
+    let mut emails = Map::new();
+    for p in &batch {
+        emails.insert(p.cid.clone(), p.item.clone());
+    }
+    let mut req = Request::new();
+    req.call(
+        "Email/import",
+        json!({ "accountId": net.account, "emails": Value::Object(emails) }),
+        "i",
+    );
+    let cids: Vec<&str> = batch.iter().map(|p| p.cid.as_str()).collect();
+    let sent = req.fits(&net.limits).and_then(|()| {
+        retry_method_call(
+            &net.client,
+            MethodCallKind::SingleObjectWrite,
+            logger,
+            || {
+                let resp = req.send_once(&net.client, &net.api)?;
+                let mr = resp.first()?;
+                check_method_error(mr)?;
+                Ok(cids
+                    .iter()
+                    .map(|cid| interpret_import_for(mr, cid, cids.len()))
+                    .collect::<Vec<_>>())
+            },
+        )
+    });
+    match sent {
+        Ok(outcomes) => {
+            for (p, outcome) in batch.into_iter().zip(outcomes) {
+                let row = &units[p.unit].row;
+                match outcome {
+                    SingleImport::Created => counts.created += 1,
+                    SingleImport::Skipped => counts.skipped += 1,
+                    SingleImport::NotCreated { error_type, .. } if error_type == "blobNotFound" => {
+                        retry_after_reupload(net, uploader, maps, &p.cid, row, counts, logger);
+                    }
+                    SingleImport::NotCreated { detail, .. } => {
+                        logger.warn(&format!(
+                            "Email/import {} ({}) failed: {detail}",
+                            p.cid,
+                            blob_hint(uploader, row)
+                        ));
+                        counts.failed += 1;
+                    }
+                }
+            }
+        }
+        Err(JmapError::RequestTooLarge | JmapError::SingleObjectTooLarge(_)) if batch.len() > 1 => {
+            let mut batch = batch;
+            let second = batch.split_off(batch.len() / 2);
+            send_batch(net, uploader, maps, units, batch, counts, logger, unclear);
+            send_batch(net, uploader, maps, units, second, counts, logger, unclear);
+        }
+        Err(e) if applied_unknown(&e) => {
+            logger.warn(&format!(
+                "Email/import of {} message(s) ended without a clear answer ({e}); the target is checked before any is sent again",
+                batch.len()
+            ));
+            unclear.extend(batch);
+        }
+        Err(JmapError::Method { .. }) if batch.len() > 1 => {
+            for p in batch {
+                send_batch(net, uploader, maps, units, vec![p], counts, logger, unclear);
+            }
+        }
+        Err(e) => {
+            for p in batch {
+                logger.warn(&format!(
+                    "Email/import {} ({}) send failed: {e}{}",
+                    p.cid,
+                    blob_hint(uploader, &units[p.unit].row),
+                    size_note(&e)
+                ));
+                counts.failed += 1;
+            }
+        }
+    }
+}
+
+/// Whether the server may have applied a call that failed with `e`: the
+/// connection broke after the request was sent, the answer was unreadable, or
+/// the server said it applied part of it.
+fn applied_unknown(e: &JmapError) -> bool {
+    match e {
+        JmapError::Transport(_)
+        | JmapError::RetriesExhausted(_)
+        | JmapError::Malformed(_)
+        | JmapError::HttpStatus { .. } => true,
+        JmapError::Method { error_type, .. } => error_type == "serverPartialFail",
+        _ => false,
+    }
+}
+
+/// Settles messages whose import ended without a clear answer: reads the
+/// target again, counts those that arrived as created, and imports the rest
+/// one at a time. If the target cannot be read, they are counted as failed --
+/// the next export matches whatever did arrive, so none is ever doubled.
+fn settle_unclear(
+    net: &Net,
+    uploader: &mut Uploader,
+    maps: &Maps,
+    units: &[Unit],
+    unclear: Vec<Pending>,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) {
+    let (targets, target_keys) = match target_emails(net) {
+        Ok(t) => t,
+        Err(e) => {
+            logger.warn(&format!(
+                "could not read the target to settle {} message(s) ({e}); they count as failed, and the next export matches whatever arrived",
+                unclear.len()
+            ));
+            counts.failed += unclear.len() as u64;
+            return;
+        }
+    };
+    let keys: Vec<EmailKey> = email_keys(
+        &unclear
+            .iter()
+            .map(|p| index_from_json(&units[p.unit].row.message_match))
+            .collect::<Vec<_>>(),
+    );
+    let sizes: Vec<Option<u64>> = unclear
+        .iter()
+        .map(|p| uploader.blob_len(units[p.unit].row.blob_local_id))
+        .collect();
+    let pairs = pair_with_targets(&keys, &sizes, &target_keys, &targets);
+    for (p, found) in unclear.into_iter().zip(pairs) {
+        if found.is_some() {
+            counts.created += 1;
+            continue;
+        }
+        let unit = &units[p.unit];
+        export_one(
+            net,
+            uploader,
+            maps,
             unit.local_id,
             &unit.row,
             counts,
             logger,
-            ),
+        );
     }
 }
-    update_batch(net, ty, updates, counts, logger);
-
-    Ok(Plan::default())
-}
 
 /// One message to write: the archive rows that hold the same bytes, folded
 /// together. A source that files one message in several folders (IMAP and
@@ -348,6 +627,15 @@ fn blob_hint(uploader: &Uploader, row: &EmailRow) -> String {
     s
 }
 
+/// A failure reason for the dry-run plan: the size message on its own, or
+/// the error as it is.
+fn plain_reason(e: &JmapError) -> String {
+    match e {
+        JmapError::SingleObjectTooLarge(m) => m.clone(),
+        other => other.to_string(),
+    }
+}
+
 fn size_note(e: &JmapError) -> &'static str {
     if matches!(
         e,
@@ -521,6 +809,13 @@ fn send_single_import(
 
 fn interpret_import(mr: &MethodCall, cid: &str) -> Result<SingleImport, JmapError> {
     check_method_error(mr)?;
+    Ok(interpret_import_for(mr, cid, 1))
+}
+
+/// One message's outcome in an `Email/import` answer. With a single message
+/// in the call, any `created` entry is taken as its own, as servers may key it
+/// differently.
+fn interpret_import_for(mr: &MethodCall, cid: &str, in_call: usize) -> SingleImport {
     if let Some(err) = mr
         .args
         .get("notCreated")
@@ -533,25 +828,21 @@ fn interpret_import(mr: &MethodCall, cid: &str) -> Result<SingleImport, JmapErro
             .unwrap_or("")
             .to_owned();
         if error_type == "alreadyExists" {
-            return Ok(SingleImport::Skipped);
+            return SingleImport::Skipped;
         }
-        return Ok(SingleImport::NotCreated {
+        return SingleImport::NotCreated {
             error_type,
             detail: err.to_string(),
-        });
+        };
     }
-    if mr
+    let created = mr.args.get("created").and_then(Value::as_object);
-        .args
+    if created.is_some_and(|c| c.contains_key(cid) || (in_call == 1 && !c.is_empty())) {
-        .get("created")
+        return SingleImport::Created;
-        .and_then(Value::as_object)
-        .is_some_and(|c| !c.is_empty())
-    {
-        return Ok(SingleImport::Created);
     }
-    Ok(SingleImport::NotCreated {
+    SingleImport::NotCreated {
         error_type: String::new(),
         detail: format!("Email/import returned neither created nor notCreated for {cid}"),
-    })
+    }
 }
 
 #[cfg(test)]

src/sync/export/flat.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -108,6 +109,10 @@ pub fn reconcile(
                             ty.jmap_name(),
                             id
                         ));
+                    } else if net.dry_run {
+                        // The object only exists in the plan; claiming the
+                        // default would be a write.
+                        default_claimed = true;
                     } else {
                         let mut req = crate::jmap::request::Request::new();
                         req.call(

src/sync/export/sieve.rs

@@ -14,6 +14,7 @@ use super::sieve_names;
 use super::{Maps, Net, Plan, Uploader};
 use crate::error::Error;
 use crate::jmap::blobxfer;
+use crate::jmap::error::JmapError;
 use crate::jmap::request::{Request, check_method_error};
 use crate::logging::{LEVEL_DEFAULT, Logger};
 use crate::sync::import_jmap::mapping::BlobBytes;
@@ -72,6 +73,7 @@ pub fn reconcile(
         .map(|(_, n, _, _)| n.clone().unwrap_or_default());
 
     let mut updates: Vec<(String, Value)> = Vec::new();
+    let mut validator = Validator::default();
 
     for (local, name, is_active, blob_local) in &locals {
         let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
@@ -90,6 +92,7 @@ pub fn reconcile(
             };
             match content_differs(net, &ours, target_blob.get(&id)) {
                 Ok(false) => counts.skipped += 1,
+                Ok(true) if !validator.accepts(net, label, &ours, counts, logger) => {}
                 Ok(true) => {
                     let blob = match &rewritten {
                         Some((bytes, renamed)) => {
@@ -116,6 +119,15 @@ pub fn reconcile(
             id
         } else {
             let cid = format!("c{local}");
+            if net.dry_run {
+                let ours = match &rewritten {
+                    Some((bytes, _)) => bytes.clone(),
+                    None => uploader.bytes(*blob_local).map_err(Error::from)?,
+                };
+                if !validator.accepts(net, label, &ours, counts, logger) {
+                    continue;
+                }
+            }
             if let Some((_, renamed)) = &rewritten {
                 log_renames(label, renamed, logger);
             }
@@ -217,6 +229,100 @@ pub fn reconcile(
     })
 }
 
+/// Checks, in a dry run, that the target would accept each script about to
+/// be written. A script too large to upload, or one `SieveScript/validate`
+/// rejects, is counted as a failure and listed in the plan. A real run
+/// checks nothing here: the target's own answer to the write is the check.
+#[derive(Default)]
+struct Validator {
+    unsupported: bool,
+}
+
+impl Validator {
+    /// Whether the script may be written. Always true outside a dry run.
+    fn accepts(
+        &mut self,
+        net: &Net,
+        label: &str,
+        bytes: &[u8],
+        counts: &mut TypeCounts,
+        logger: &Logger,
+    ) -> bool {
+        if !net.dry_run || self.unsupported {
+            return true;
+        }
+        let why = match net.check_upload_size(bytes.len() as u64) {
+            Err(JmapError::SingleObjectTooLarge(m)) => Some(m),
+            Err(e) => Some(e.to_string()),
+            Ok(()) => match validate(net, bytes) {
+                Ok(why) => why,
+                Err(e) if is_unknown_method(&e) => {
+                    logger.warn("the target cannot validate Sieve scripts; they are not checked");
+                    self.unsupported = true;
+                    None
+                }
+                Err(e) => {
+                    logger.warn(&format!("SieveScript {label}: not validated: {e}"));
+                    None
+                }
+            },
+        };
+        match why {
+            None => true,
+            Some(why) => {
+                logger.warn(&format!(
+                    "SieveScript {label}: the target would reject it: {why}"
+                ));
+                net.would_fail(format!(
+                    "SieveScript \"{label}\": the target would reject it: {why}"
+                ));
+                counts.failed += 1;
+                false
+            }
+        }
+    }
+}
+
+/// Asks the target whether it would accept `bytes` as a Sieve script: `None`
+/// if it would, or its reason. The script goes up as a blob, which the
+/// server keeps only for a while; nothing is created in the account.
+fn validate(net: &Net, bytes: &[u8]) -> Result<Option<String>, JmapError> {
+    let blob = blobxfer::upload_bytes(
+        &net.client,
+        &net.session,
+        &net.account,
+        "application/sieve",
+        bytes,
+    )?;
+    let mut req = Request::new();
+    req.call(
+        "SieveScript/validate",
+        json!({ "accountId": net.account, "blobId": blob.0 }),
+        "v",
+    );
+    let resp = req.send(&net.client, &net.api)?;
+    let mr = resp.by_call_id("v")?;
+    check_method_error(mr)?;
+    Ok(match mr.args.get("error") {
+        None | Some(Value::Null) => None,
+        Some(err) => Some(
+            err.get("description")
+                .and_then(Value::as_str)
+                .or_else(|| err.get("type").and_then(Value::as_str))
+                .unwrap_or("rejected")
+                .to_owned(),
+        ),
+    })
+}
+
+fn is_unknown_method(e: &JmapError) -> bool {
+    match e {
+        JmapError::UnknownMethod => true,
+        JmapError::Method { error_type, .. } => error_type == "unknownMethod",
+        _ => false,
+    }
+}
+
 /// The target's `sieveExtensions`, from its Sieve account capability.
 fn target_sieve_extensions(net: &Net) -> Vec<String> {
     net.session

src/sync/export/uidtype.rs

@@ -17,6 +17,7 @@ use crate::logging::Logger;
 use crate::sync::import_jmap::mapping::{
     CALENDAR_EVENT_SELECT, CONTACT_CARD_SELECT, calendar_event_to_wire, contact_card_to_wire,
 };
+use crate::sync::progress::Progress;
 use crate::sync::prune::{TargetObj, candidates};
 use crate::sync::{Context, TypeCounts};
 use crate::types::ObjectType;
@@ -79,7 +80,13 @@ pub fn reconcile(
     let mut matched_uids: HashSet<String> = HashSet::new();
     let mut updates: Vec<(String, Value)> = Vec::new();
     let blobs = Uploader::new(net, &ctx.conn);
+    let mut progress = Progress::new(
+        format!("export: {}", ty.jmap_name()),
+        rows.len() as u64,
+        logger,
+    );
     for (local, uid) in &rows {
+        progress.add(1);
         if let Some((tid, existing)) = by_uid.get(uid) {
             maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
             matched_uids.insert(uid.clone());
@@ -102,6 +109,7 @@ pub fn reconcile(
             Err(e) if e.aborts_run() => return Err(e),
             Err(e) => {
                 logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
+                net.would_fail(format!("{}: {e}", describe(ty, *local, uid)));
                 counts.failed += 1;
                 continue;
             }

src/sync/import_exchange_ews/calendar.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -104,7 +105,12 @@ fn reconcile_one(
         to_fetch.push(id.clone());
     }
     if !to_fetch.is_empty() {
-        let failed_items = for_each_fetched_item(ctx, ItemShape::CalendarItem, &to_fetch, |msg| {
+        let failed_items = for_each_fetched_item(
+            ctx,
+            ItemShape::CalendarItem,
+            &to_fetch,
+            &outcome.sizes,
+            |msg| {
                 if !msg.success {
                     if matches!(
                         msg.response_code,
@@ -144,7 +150,8 @@ fn reconcile_one(
                     existing,
                     counts,
                 )
-        })?;
+            },
+        )?;
         counts.failed += failed_items;
     }
     delete_vanished(

src/sync/import_exchange_ews/contacts.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -96,7 +97,8 @@ fn reconcile_one(
         to_fetch.push(id.clone());
     }
     if !to_fetch.is_empty() {
-        let failed_items = for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, |msg| {
+        let failed_items =
+            for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, &outcome.sizes, |msg| {
                 if !msg.success {
                     if matches!(
                         msg.response_code,

src/sync/import_exchange_ews/coordinator.rs

@@ -37,6 +37,8 @@ pub struct EwsImportConfig {
     pub auth: EwsAuth,
     pub ews_connections: usize,
     pub getitem_batch: usize,
+    /// Byte cap for one GetItem batch; see `sync::batch`.
+    pub getitem_batch_bytes: u64,
     pub attachment_batch: usize,
     pub use_syncfolderitems: bool,
     pub allow_source_change: bool,
@@ -147,6 +149,7 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
         url: &session_url,
         source_id,
         batch_size: config.getitem_batch.max(1),
+        batch_bytes: config.getitem_batch_bytes.max(1),
         attachment_batch: config.attachment_batch.max(1),
         connections: config.ews_connections.clamp(1, 8),
         use_syncfolderitems: config.use_syncfolderitems,

src/sync/import_exchange_ews/items.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -23,6 +24,8 @@ pub struct ItemRunCtx<'a> {
     pub url: &'a str,
     pub source_id: i64,
     pub batch_size: usize,
+    /// Byte cap for one GetItem batch, by `item:Size`; see `sync::batch`.
+    pub batch_bytes: u64,
     pub attachment_batch: usize,
     pub connections: usize,
     pub use_syncfolderitems: bool,
@@ -47,6 +50,9 @@ pub struct EnumeratedItem {
 pub struct EnumerationOutcome {
     pub items: Vec<EnumeratedItem>,
     pub mode: EnumerationMode,
+    /// `item:Size` by item id, where the listing gave it. FindItem does;
+    /// SyncFolderItems changes do not, and are batched by count.
+    pub sizes: HashMap<String, u64>,
 }
 
 #[derive(Debug, Clone)]
@@ -70,9 +76,10 @@ pub fn enumerate_folder(
     {
         return Ok(outcome);
     }
-    enumerate_via_find_item(ctx, folder).map(|items| EnumerationOutcome {
+    enumerate_via_find_item(ctx, folder).map(|(items, sizes)| EnumerationOutcome {
         items,
         mode: EnumerationMode::Full,
+        sizes,
     })
 }
 
@@ -151,14 +158,16 @@ fn try_sync_folder_items(
             deletions,
             new_sync_state: sync_state,
         },
+        sizes: HashMap::new(),
     }))
 }
 
 fn enumerate_via_find_item(
     ctx: &ItemRunCtx<'_>,
     folder: &FolderId,
-) -> Result<Vec<EnumeratedItem>, EwsError> {
+) -> Result<(Vec<EnumeratedItem>, HashMap<String, u64>), EwsError> {
     let mut items: Vec<EnumeratedItem> = Vec::new();
+    let mut sizes: HashMap<String, u64> = HashMap::new();
     let mut offset: u32 = 0;
     let page_size: u32 = 500;
     loop {
@@ -172,6 +181,9 @@ fn enumerate_via_find_item(
         let parsed = parse_find_item_response(&resp.body)?;
         let returned = parsed.items.len() as u32;
         for entry in parsed.items {
+            if let Some(size) = entry.size {
+                sizes.insert(entry.id.id.clone(), size);
+            }
             items.push(EnumeratedItem {
                 element: entry.element,
                 id: entry.id,
@@ -188,7 +200,7 @@ fn enumerate_via_find_item(
             break;
         }
     }
-    Ok(items)
+    Ok((items, sizes))
 }
 
 #[derive(Debug, Clone)]
@@ -285,13 +297,22 @@ pub fn get_items(
     shape: ItemShape,
     ids: &[ItemId],
 ) -> Result<GetItemBatchOutcome, EwsError> {
-    let batch = ctx.batch_size.max(1);
+    let batches: Vec<&[ItemId]> = ids.chunks(ctx.batch_size.max(1)).collect();
+    get_item_batches(ctx, shape, &batches)
+}
+
+/// Runs one GetItem per batch, over up to `connections` connections.
+pub fn get_item_batches(
+    ctx: &ItemRunCtx<'_>,
+    shape: ItemShape,
+    batches: &[&[ItemId]],
+) -> Result<GetItemBatchOutcome, EwsError> {
     let workers = ctx.connections.clamp(1, 8);
     let version = ctx.client.server_version();
     let mut failed_items: u64 = 0;
-    if workers <= 1 || ids.len() <= batch {
+    if workers <= 1 || batches.len() <= 1 {
         let mut all = Vec::new();
-        for chunk in ids.chunks(batch) {
+        for chunk in batches.iter().copied() {
             let body = get_item_body(shape, chunk, version);
             match ctx.client.call(ctx.url, "GetItem", &body) {
                 Ok(resp) => match parse_response_messages(&resp.body, "GetItemResponseMessage") {
@@ -337,7 +358,7 @@ pub fn get_items(
             (n, result)
         });
     let mut submitted = 0usize;
-    for chunk in ids.chunks(batch) {
+    for chunk in batches {
         pool.submit(chunk.to_vec());
         submitted += 1;
     }
@@ -368,21 +389,31 @@ pub fn get_items(
     })
 }
 
+/// Fetches `ids` in GetItem batches of at most `batch_size` items and, where
+/// `sizes` knows them, at most `batch_bytes` bytes, and hands each message
+/// on as it is parsed. Batches are fetched `connections` at a time, so what
+/// is held at once is about `batch_bytes` per connection, however large the
+/// mail is.
 pub fn for_each_fetched_item<F>(
     ctx: &ItemRunCtx<'_>,
     shape: ItemShape,
     ids: &[ItemId],
+    sizes: &HashMap<String, u64>,
     mut on_message: F,
 ) -> Result<u64, Error>
 where
     F: FnMut(crate::exchange_ews::parse::ResponseMessage) -> Result<(), Error>,
 {
-    let batch = ctx.batch_size.max(1);
     let workers = ctx.connections.clamp(1, 8);
-    let window = batch.saturating_mul(workers).max(batch);
+    let batches = crate::sync::batch::by_count_and_bytes(
+        ids,
+        |id| sizes.get(&id.id).copied().unwrap_or(0),
+        ctx.batch_size,
+        ctx.batch_bytes,
+    );
     let mut failed_items = 0u64;
-    for win in ids.chunks(window) {
+    for win in batches.chunks(workers) {
-        let outcome = get_items(ctx, shape, win).map_err(Error::from)?;
+        let outcome = get_item_batches(ctx, shape, win).map_err(Error::from)?;
         failed_items = failed_items.saturating_add(outcome.failed_items);
         for msg in outcome.messages {
             on_message(msg)?;
@@ -444,6 +475,7 @@ mod tests {
                 element: "Message".to_owned(),
                 id: ItemId::new("A", "ck-2"),
             }],
+            sizes: HashMap::new(),
             mode: EnumerationMode::Delta {
                 deletions: vec!["Z".to_owned()],
                 new_sync_state: "STATE2".to_owned(),

src/sync/import_exchange_ews/messages.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ * SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -95,7 +96,8 @@ fn reconcile_one_folder(
         to_fetch.push(id.clone());
     }
     if !to_fetch.is_empty() {
-        let failed_items = for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, |msg| {
+        let failed_items =
+            for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, &outcome.sizes, |msg| {
                 if !msg.success {
                     if matches!(
                         msg.response_code,

src/sync/import_imap/coordinator.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -45,7 +46,11 @@ const EMAIL_TYPE: &str = "email";
 #[derive(Clone, Copy)]
 pub(super) struct RunOpts {
     source_id: i64,
+    /// The folder generation this folder's fetch jobs carry; see
+    /// `WorkerPool::cancel_before`.
+    generation: u64,
     fetch_batch: usize,
+    fetch_batch_bytes: u64,
     include_deleted: bool,
     logger: Logger,
 }
@@ -195,6 +200,8 @@ pub struct ImapImportConfig {
     pub automap: bool,
     pub include_deleted: bool,
     pub fetch_batch: usize,
+    /// Byte cap for one fetch chunk, by RFC822.SIZE; see `sync::batch`.
+    pub fetch_batch_bytes: u64,
     pub imap_connections: usize,
     pub allow_source_change: bool,
 }
@@ -398,7 +405,9 @@ fn run_into(
 
     let opts = RunOpts {
         source_id,
+        generation: 0,
         fetch_batch: config.fetch_batch.max(1),
+        fetch_batch_bytes: config.fetch_batch_bytes.max(1),
         include_deleted: config.include_deleted,
         logger,
     };
@@ -422,13 +431,17 @@ fn run_into(
         if i > 0 {
             let _ = control_run_collect(&mut client, &control_ctx, "NOOP");
         }
+        // Each folder gets a new generation; anything still queued or in
+        // flight for an earlier folder is skipped or dropped from here on.
+        let generation = i as u64 + 1;
+        pool.cancel_before(generation);
         match reconcile_folder(
             &mut conn,
             &mut client,
             &control_ctx,
             &pool,
             folder,
-            opts,
+            RunOpts { generation, ..opts },
             &mut email_counts,
         ) {
             Ok(()) => {}
@@ -695,7 +708,9 @@ fn reconcile_folder(
 ) -> Result<(), Error> {
     let RunOpts {
         source_id,
+        generation,
         fetch_batch,
+        fetch_batch_bytes: _,
         include_deleted: _,
         logger,
     } = opts;
@@ -801,10 +816,17 @@ fn reconcile_folder(
                     folder.name
                 ))
             })?;
-        let batches: Vec<&[u32]> = chunks(&diff.new, fetch_batch);
+        let sizes = fetch_sizes(client, control_ctx, &folder.name, &diff.new, logger);
+        let batches: Vec<&[u32]> = crate::sync::batch::by_count_and_bytes(
+            &diff.new,
+            |uid| sizes.get(uid).copied().unwrap_or(0),
+            fetch_batch,
+            opts.fetch_batch_bytes,
+        );
         let n_batches = batches.len();
         for batch in &batches {
             pool.submit(FetchJob {
+                generation,
                 folder: folder.name.clone(),
                 wire_name: folder.wire_name.clone(),
                 uidvalidity,
@@ -813,12 +835,16 @@ fn reconcile_folder(
         }
         let keepalive_interval = std::time::Duration::from_secs(45);
         let mut chunks_done: usize = 0;
-        let tx = conn.transaction()?;
         let target = FetchTarget {
             folder: folder.name.as_str(),
             uidvalidity,
             mailbox_local,
         };
+        // Committed after every chunk, not once per folder: each message is
+        // written whole (see `insert_recording_failure`), so what is
+        // committed is always consistent, a crash keeps it, and the next run
+        // fetches only the UIDs that are still missing.
+        let mut tx = conn.transaction()?;
         while chunks_done < n_batches {
             let event = loop {
                 match pool.recv_timeout(keepalive_interval) {
@@ -831,15 +857,15 @@ fn reconcile_folder(
                     }
                 }
             };
-            match event {
+            match route_event(event, generation) {
-                FetchEvent::Item { attrs, .. } => {
+                Routed::Stale => {}
-                    insert_single_message(&tx, &target, &attrs, opts, counts)?;
+                Routed::Item(attrs) => {
+                    insert_recording_failure(&mut tx, &target, &attrs, opts, counts)?;
                 }
-                FetchEvent::ChunkDone {
+                Routed::ChunkDone {
                     folder: chunk_folder,
-                    outcome,
                     uids_requested,
-                    ..
+                    outcome,
                 } => {
                     chunks_done += 1;
                     if let Err(e) = outcome {
@@ -850,6 +876,8 @@ fn reconcile_folder(
                         );
                         counts.failed += uids_requested.len() as u64;
                     }
+                    tx.commit()?;
+                    tx = conn.transaction()?;
                 }
             }
         }
@@ -875,6 +903,47 @@ fn reconcile_folder(
     Ok(())
 }
 
+/// RFC822.SIZE for each of `uids`, fetched on the control connection in
+/// large metadata-only chunks, so the body fetch can be split by bytes.
+/// Best effort: a server that won't answer just leaves the chunks sized by
+/// count.
+fn fetch_sizes(
+    client: &mut ImapClient,
+    ctx: &ControlCtx,
+    folder: &str,
+    uids: &[u32],
+    logger: Logger,
+) -> HashMap<u32, u64> {
+    const SIZE_CHUNK: usize = 1000;
+    let mut sizes = HashMap::with_capacity(uids.len());
+    for chunk in chunks(uids, SIZE_CHUNK) {
+        let set = command::format_uid_set(chunk, true);
+        let cmd = command::uid_fetch(&set, &["UID", "RFC822.SIZE"]);
+        match call_with_retry(client, ctx, |c| c.run_collect(&cmd)) {
+            Ok(resp) => {
+                for u in &resp.untagged {
+                    if let Some(attrs) = fetch::extract(u)
+                        && let (Some(uid), Some(size)) = (attrs.uid, attrs.size)
+                    {
+                        sizes.insert(uid, size);
+                    }
+                }
+            }
+            Err(e) => {
+                log_at(
+                    logger,
+                    LEVEL_DEFAULT,
+                    &format!(
+                        "folder {folder:?}: message sizes unavailable ({e}); fetching in chunks by count only"
+                    ),
+                );
+                break;
+            }
+        }
+    }
+    sizes
+}
+
 fn wipe_folder_emails(
     conn: &mut Connection,
     source_id: i64,
@@ -920,6 +989,78 @@ fn delete_vanished_emails(
     Ok(())
 }
 
+pub(super) enum Routed {
+    /// From an earlier folder generation: dropped, never filed here.
+    Stale,
+    Item(fetch::FetchAttrs),
+    ChunkDone {
+        folder: String,
+        uids_requested: Vec<u32>,
+        outcome: Result<(), ImapError>,
+    },
+}
+
+pub(super) fn route_event(event: FetchEvent, generation: u64) -> Routed {
+    match event {
+        FetchEvent::Item {
+            generation: g,
+            attrs,
+            ..
+        } if g == generation => Routed::Item(attrs),
+        FetchEvent::ChunkDone {
+            generation: g,
+            folder,
+            uids_requested,
+            outcome,
+            ..
+        } if g == generation => Routed::ChunkDone {
+            folder,
+            uids_requested,
+            outcome,
+        },
+        _ => Routed::Stale,
+    }
+}
+
+/// Writes one message inside its own savepoint. A message that cannot be
+/// imported -- an INTERNALDATE that will not parse, say -- is rolled back
+/// on its own, logged with its folder and UID, counted as failed, and the
+/// folder carries on; it stays out of the UID map, so the next run tries
+/// it again. Archive and I/O errors still stop the run.
+fn insert_recording_failure(
+    tx: &mut rusqlite::Transaction<'_>,
+    target: &FetchTarget<'_>,
+    attrs: &fetch::FetchAttrs,
+    opts: RunOpts,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let sp = tx.savepoint()?;
+    match insert_single_message(&sp, target, attrs, opts, counts) {
+        Ok(()) => {
+            sp.commit()?;
+            Ok(())
+        }
+        Err(e) if e.aborts_run() => Err(e),
+        Err(e) => {
+            drop(sp);
+            log_at(
+                opts.logger,
+                LEVEL_DEFAULT,
+                &format!(
+                    "folder {:?} uid {}: not imported: {e}",
+                    target.folder,
+                    attrs
+                        .uid
+                        .map(|u| u.to_string())
+                        .unwrap_or_else(|| "?".to_owned())
+                ),
+            );
+            counts.failed += 1;
+            Ok(())
+        }
+    }
+}
+
 pub(super) struct FetchTarget<'a> {
     pub folder: &'a str,
     pub uidvalidity: u32,
@@ -927,7 +1068,7 @@ pub(super) struct FetchTarget<'a> {
 }
 
 fn insert_single_message(
-    tx: &rusqlite::Transaction<'_>,
+    tx: &Connection,
     target: &FetchTarget<'_>,
     attrs: &fetch::FetchAttrs,
     opts: RunOpts,
@@ -935,7 +1076,9 @@ fn insert_single_message(
 ) -> Result<(), Error> {
     let RunOpts {
         source_id,
+        generation: _,
         fetch_batch: _,
+        fetch_batch_bytes: _,
         include_deleted,
         logger,
     } = opts;
@@ -1012,7 +1155,9 @@ fn refresh_present_flags(
 ) -> Result<u64, Error> {
     let RunOpts {
         source_id,
+        generation: _,
         fetch_batch,
+        fetch_batch_bytes: _,
         include_deleted,
         logger: _,
     } = opts;
@@ -1237,6 +1382,36 @@ fn dry_run_summary(
 mod tests {
     use super::*;
 
+    fn item(generation: u64, uid: u32) -> FetchEvent {
+        FetchEvent::Item {
+            generation,
+            folder: "F".to_owned(),
+            uidvalidity: 1,
+            attrs: fetch::FetchAttrs {
+                uid: Some(uid),
+                ..Default::default()
+            },
+        }
+    }
+
+    #[test]
+    fn events_from_another_generation_are_dropped() {
+        assert!(matches!(route_event(item(1, 5), 2), Routed::Stale));
+        assert!(matches!(route_event(item(3, 5), 2), Routed::Stale));
+        match route_event(item(2, 5), 2) {
+            Routed::Item(attrs) => assert_eq!(attrs.uid, Some(5)),
+            _ => panic!("current-generation item was not routed"),
+        }
+        let stale_done = FetchEvent::ChunkDone {
+            generation: 1,
+            folder: "Old".to_owned(),
+            uidvalidity: 1,
+            uids_requested: vec![5],
+            outcome: Ok(()),
+        };
+        assert!(matches!(route_event(stale_done, 2), Routed::Stale));
+    }
+
     #[test]
     fn parse_endpoint_imaps_defaults_to_993() {
         let e = parse_endpoint("imaps://mail.example.com").unwrap();

src/sync/import_imap/internaldate.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -58,20 +59,22 @@ pub fn imap_internaldate_to_rfc3339(s: &str) -> Result<String, Error> {
     Ok(out)
 }
 
+// RFC 3501 spells the month "Jan", but servers are not all that careful,
+// and a date is not worth losing a message over: match any case.
 fn month_to_num(s: &str) -> Result<u32, Error> {
-    let m = match s {
+    let m = match s.to_ascii_lowercase().as_str() {
-        "Jan" => 1,
+        "jan" => 1,
-        "Feb" => 2,
+        "feb" => 2,
-        "Mar" => 3,
+        "mar" => 3,
-        "Apr" => 4,
+        "apr" => 4,
-        "May" => 5,
+        "may" => 5,
-        "Jun" => 6,
+        "jun" => 6,
-        "Jul" => 7,
+        "jul" => 7,
-        "Aug" => 8,
+        "aug" => 8,
-        "Sep" => 9,
+        "sep" => 9,
-        "Oct" => 10,
+        "oct" => 10,
-        "Nov" => 11,
+        "nov" => 11,
-        "Dec" => 12,
+        "dec" => 12,
         other => return Err(Error::Partial(format!("INTERNALDATE month {other:?}"))),
     };
     Ok(m)
@@ -99,6 +102,22 @@ fn parse_zone(s: &str) -> Result<(char, u32, u32), Error> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn month_matches_any_case() {
+        for d in [
+            "12-May-2025 10:00:00 +0000",
+            "12-may-2025 10:00:00 +0000",
+            "12-MAY-2025 10:00:00 +0000",
+        ] {
+            assert_eq!(
+                imap_internaldate_to_rfc3339(d).unwrap(),
+                "2025-05-12T10:00:00Z",
+                "{d}"
+            );
+        }
+        assert!(imap_internaldate_to_rfc3339("12-Mai-2025 10:00:00 +0000").is_err());
+    }
+
     #[test]
     fn utc_zone_becomes_z() {
         assert_eq!(

src/sync/import_imap/pool.rs

@@ -1,14 +1,16 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
 
 use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::thread;
 
-use crossbeam_channel::{Receiver, Sender, unbounded};
+use crossbeam_channel::{Receiver, Sender, bounded, unbounded};
 
 use crate::imap::client::{ConnectMode, ImapClient};
 use crate::imap::command;
@@ -23,7 +25,12 @@ use super::fetch::FetchAttrs;
 
 pub const HARD_CAP: usize = 8;
 
+/// A job or event from an older folder generation than the one the
+/// coordinator is working on belongs to a folder it has already given up
+/// on. Workers skip such jobs without fetching, and the coordinator drops
+/// such events, so nothing from one folder can be filed into the next.
 pub struct FetchJob {
+    pub generation: u64,
     pub folder: String,
     pub wire_name: String,
     pub uidvalidity: u32,
@@ -32,11 +39,13 @@ pub struct FetchJob {
 
 pub enum FetchEvent {
     Item {
+        generation: u64,
         folder: String,
         uidvalidity: u32,
         attrs: FetchAttrs,
     },
     ChunkDone {
+        generation: u64,
         folder: String,
         uidvalidity: u32,
         uids_requested: Vec<u32>,
@@ -59,22 +68,29 @@ pub struct WorkerPool {
     job_tx: Sender<FetchJob>,
     event_rx: Receiver<FetchEvent>,
     handles: Vec<thread::JoinHandle<()>>,
+    cancel_below: Arc<AtomicU64>,
 }
 
 impl WorkerPool {
     pub fn start(args: WorkerArgs, pool_size: usize) -> Result<WorkerPool, ImapError> {
         let size = pool_size.clamp(1, HARD_CAP);
         let (job_tx, job_rx) = unbounded::<FetchJob>();
-        let (event_tx, event_rx) = unbounded::<FetchEvent>();
+        // Jobs are only lists of UIDs, but each event carries a whole message:
+        // bounding the events stops fast workers from running ahead of the
+        // single archive writer, so memory holds at most a couple of messages
+        // per worker rather than whole folders.
+        let (event_tx, event_rx) = bounded::<FetchEvent>(size * 2);
         let mut handles = Vec::with_capacity(size);
         let args = Arc::new(args);
+        let cancel_below = Arc::new(AtomicU64::new(0));
 
         for _ in 0..size {
             let args = args.clone();
             let job_rx = job_rx.clone();
             let event_tx = event_tx.clone();
+            let cancel_below = cancel_below.clone();
             let handle = thread::spawn(move || {
-                worker_loop(args, job_rx, event_tx);
+                worker_loop(args, job_rx, event_tx, cancel_below);
             });
             handles.push(handle);
         }
@@ -83,9 +99,17 @@ impl WorkerPool {
             job_tx,
             event_rx,
             handles,
+            cancel_below,
         })
     }
 
+    /// Jobs of any generation below `generation` are skipped from now on:
+    /// called when the coordinator moves to a new folder, so work still
+    /// queued for one it abandoned is not fetched.
+    pub fn cancel_before(&self, generation: u64) {
+        self.cancel_below.fetch_max(generation, Ordering::SeqCst);
+    }
+
     pub fn submit(&self, job: FetchJob) {
         let _ = self.job_tx.send(job);
     }
@@ -101,21 +125,43 @@ impl WorkerPool {
         self.event_rx.recv_timeout(timeout)
     }
 
+    /// Stops the workers. Events still in flight are drained and dropped
+    /// first: a worker blocked handing over an event the coordinator will
+    /// never read (after a folder was abandoned) would otherwise never
+    /// finish, and joining it would hang.
     pub fn shutdown(self) {
+        self.cancel_before(u64::MAX);
         drop(self.job_tx);
+        while self.event_rx.recv().is_ok() {}
         for h in self.handles {
             let _ = h.join();
         }
     }
 }
 
-fn worker_loop(args: Arc<WorkerArgs>, job_rx: Receiver<FetchJob>, event_tx: Sender<FetchEvent>) {
+fn worker_loop(
+    args: Arc<WorkerArgs>,
+    job_rx: Receiver<FetchJob>,
+    event_tx: Sender<FetchEvent>,
+    cancel_below: Arc<AtomicU64>,
+) {
     let mut client: Option<ImapClient> = None;
     let mut current_folder: Option<String> = None;
     while let Ok(job) = job_rx.recv() {
+        let job_gen = job.generation;
         let job_folder = job.folder.clone();
         let job_uv = job.uidvalidity;
         let job_uids = job.uids.clone();
+        if job_gen < cancel_below.load(Ordering::SeqCst) {
+            let _ = event_tx.send(FetchEvent::ChunkDone {
+                generation: job_gen,
+                folder: job_folder,
+                uidvalidity: job_uv,
+                uids_requested: job_uids,
+                outcome: Err(ImapError::Protocol("cancelled: folder abandoned".into())),
+            });
+            continue;
+        }
         let event_tx_for_job = event_tx.clone();
         let outcome = match catch_unwind(AssertUnwindSafe(|| {
             run_job_with_retry(
@@ -134,6 +180,7 @@ fn worker_loop(args: Arc<WorkerArgs>, job_rx: Receiver<FetchJob>, event_tx: Send
             }
         };
         let _ = event_tx.send(FetchEvent::ChunkDone {
+            generation: job_gen,
             folder: job_folder,
             uidvalidity: job_uv,
             uids_requested: job_uids,
@@ -235,6 +282,7 @@ fn run_one_job(
         *current_folder = Some(job.folder.clone());
     }
     let set = command::format_uid_set(&job.uids, true);
+    let generation = job.generation;
     let folder = job.folder.clone();
     let uv = job.uidvalidity;
     client.run_streamed(
@@ -247,6 +295,7 @@ fn run_one_job(
                 && let Some(attrs) = super::fetch::extract(&u)
             {
                 let _ = event_tx.send(FetchEvent::Item {
+                    generation,
                     folder: folder.clone(),
                     uidvalidity: uv,
                     attrs,
@@ -256,3 +305,76 @@ fn run_one_job(
     )?;
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+
+    fn unreachable_args() -> WorkerArgs {
+        WorkerArgs {
+            connector: Arc::new(Connector::new(false).expect("connector")),
+            endpoint: Arc::new(Endpoint {
+                host: "127.0.0.1".to_owned(),
+                port: 1,
+                implicit_tls: false,
+            }),
+            mode: ConnectMode::Plain,
+            auth: ImapAuth::Basic {
+                user: "u".to_owned(),
+                password: "p".to_owned(),
+            },
+            compress: false,
+            policy: RetryPolicy::new(0),
+            backoff: BackoffState::new(),
+            logger: Logger::from_flags(false, 0),
+        }
+    }
+
+    #[test]
+    fn a_job_from_an_abandoned_generation_is_skipped_without_fetching() {
+        // Port 1 refuses connections: a job that were actually run would come
+        // back as a connection error, not as a cancellation.
+        let pool = WorkerPool::start(unreachable_args(), 1).expect("pool");
+        pool.cancel_before(2);
+        pool.submit(FetchJob {
+            generation: 1,
+            folder: "Old".to_owned(),
+            wire_name: "Old".to_owned(),
+            uidvalidity: 7,
+            uids: vec![1, 2, 3],
+        });
+        match pool.recv_timeout(Duration::from_secs(5)).expect("event") {
+            FetchEvent::ChunkDone {
+                generation,
+                folder,
+                uids_requested,
+                outcome,
+                ..
+            } => {
+                assert_eq!(generation, 1);
+                assert_eq!(folder, "Old");
+                assert_eq!(uids_requested, vec![1, 2, 3]);
+                let err = outcome.expect_err("cancelled");
+                assert!(err.to_string().contains("cancelled"), "{err}");
+            }
+            FetchEvent::Item { .. } => panic!("a cancelled job fetched something"),
+        }
+        pool.shutdown();
+    }
+
+    #[test]
+    fn shutdown_returns_with_work_still_queued() {
+        let pool = WorkerPool::start(unreachable_args(), 2).expect("pool");
+        for g in 0..20u64 {
+            pool.submit(FetchJob {
+                generation: g,
+                folder: format!("F{g}"),
+                wire_name: format!("F{g}"),
+                uidvalidity: 1,
+                uids: vec![1],
+            });
+        }
+        pool.shutdown();
+    }
+}

src/sync/import_maildir/messages.rs

@@ -1,5 +1,6 @@
 /*
  * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -140,7 +141,7 @@ pub struct InsertContext<'a> {
 }
 
 pub fn insert_new(
-    tx: &rusqlite::Transaction<'_>,
+    tx: &Connection,
     ctx: InsertContext<'_>,
     entry: &DiskEntry,
 ) -> Result<Option<i64>, InsertError> {
@@ -268,6 +269,10 @@ pub fn delete_vanished(
 
 const PROGRESS_TICK: u64 = 1000;
 
+/// New messages are committed in groups of this many rather than once per
+/// folder, so an interrupted import of a large folder keeps what it wrote.
+const COMMIT_EVERY: u64 = 500;
+
 pub fn apply_folder(
     conn: &mut Connection,
     ctx: InsertContext<'_>,
@@ -277,15 +282,23 @@ pub fn apply_folder(
 ) -> Result<(), crate::error::Error> {
     let stored_keywords =
         load_present_keywords(conn, ctx.source_id, ctx.folder).unwrap_or_default();
-    let tx = conn.transaction()?;
+    let mut tx = conn.transaction()?;
     let total_new = diff.new.len() as u64;
     let mut inserted: u64 = 0;
     for entry in &diff.new {
-        match insert_new(&tx, ctx, entry) {
+        // Each message in its own savepoint: one that fails part-way leaves
+        // nothing behind, not an email row without its id mapping.
+        let sp = tx.savepoint()?;
+        match insert_new(&sp, ctx, entry) {
             Ok(Some(_)) => {
+                sp.commit()?;
                 counts.created += 1;
                 counts.fetched += 1;
                 inserted += 1;
+                if inserted.is_multiple_of(COMMIT_EVERY) {
+                    tx.commit()?;
+                    tx = conn.transaction()?;
+                }
                 if inserted.is_multiple_of(PROGRESS_TICK)
                     && logger.enabled(crate::logging::LEVEL_PROGRESS)
                 {
@@ -296,9 +309,11 @@ pub fn apply_folder(
                 }
             }
             Ok(None) => {
+                sp.commit()?;
                 counts.skipped += 1;
             }
             Err(e) => {
+                drop(sp);
                 logger.warn(&format!(
                     "maildir {folder:?}/{name}: {e}",
                     folder = ctx.folder,
@@ -423,6 +438,43 @@ mod tests {
         }
     }
 
+    #[test]
+    fn a_message_that_cannot_be_read_is_recorded_and_the_rest_are_imported() {
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"Subject: a\r\n\r\na");
+        let gone = write_maildir_message(td.path(), "cur", "2.M0.host:2,S", b"Subject: b\r\n\r\nb");
+        write_maildir_message(td.path(), "cur", "3.M0.host:2,S", b"Subject: c\r\n\r\nc");
+        let listing = list_folder(td.path()).unwrap();
+        // Gone between the listing and the read, as a file being moved is.
+        fs::remove_file(gone).unwrap();
+        let (mut c, sid) = fresh_archive();
+        let d = diff(listing.entries, &HashMap::new());
+        let ctx = InsertContext {
+            source_id: sid,
+            folder: "INBOX",
+            mailbox_local: 1,
+            include_deleted: false,
+        };
+        let mut counts = TypeCounts::default();
+        apply_folder(
+            &mut c,
+            ctx,
+            d,
+            &mut counts,
+            crate::logging::Logger::from_flags(false, 0),
+        )
+        .unwrap();
+        assert_eq!((counts.created, counts.failed), (2, 1));
+        let emails: i64 = c
+            .query_row("SELECT COUNT(*) FROM emails", [], |r| r.get(0))
+            .unwrap();
+        let mapped: i64 = c
+            .query_row("SELECT COUNT(*) FROM sync_id_maildir", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!((emails, mapped), (2, 2), "no email row without its mapping");
+    }
+
     #[test]
     fn list_folder_returns_cur_and_new_skipping_tmp() {
         let td = tempfile::tempdir().unwrap();

src/sync/mod.rs

@@ -5,6 +5,7 @@
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
 
+pub mod batch;
 pub mod emailmeta;
 pub mod export;
 pub mod import_dav;
@@ -16,6 +17,7 @@ pub mod import_maildir;
 pub mod import_managesieve;
 pub mod import_takeout;
 pub mod keys;
+pub mod progress;
 pub mod prune;
 
 use std::path::PathBuf;

src/sync/progress.rs

@@ -0,0 +1,196 @@
+/*
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+//! A progress line for long runs: how many of how many, how fast, and about
+//! how long is left. Printed to stderr at the default log level, at most once
+//! per interval, so a large export shows it is moving without flooding the
+//! terminal.
+
+use std::time::{Duration, Instant};
+
+use crate::logging::{LEVEL_DEFAULT, Logger};
+
+/// How often a progress line is printed while work continues.
+pub const PROGRESS_INTERVAL: Duration = Duration::from_secs(5);
+
+pub struct Progress {
+    label: String,
+    total: u64,
+    done: u64,
+    started: Instant,
+    last: Instant,
+    interval: Duration,
+    enabled: bool,
+}
+
+impl Progress {
+    /// `label` names the work, e.g. "export: Email". Nothing is printed when
+    /// the logger is quiet or there is nothing to do.
+    pub fn new(label: impl Into<String>, total: u64, logger: &Logger) -> Progress {
+        let now = Instant::now();
+        Progress {
+            label: label.into(),
+            total,
+            done: 0,
+            started: now,
+            last: now,
+            interval: PROGRESS_INTERVAL,
+            enabled: logger.enabled(LEVEL_DEFAULT) && total > 0,
+        }
+    }
+
+    /// Records `n` more items done, and prints a line once the interval has
+    /// passed since the last one.
+    pub fn add(&mut self, n: u64) {
+        self.done = (self.done + n).min(self.total);
+        if !self.enabled {
+            return;
+        }
+        let now = Instant::now();
+        if now.duration_since(self.last) >= self.interval {
+            self.last = now;
+            eprintln!("{}", self.line(now.duration_since(self.started)));
+        }
+    }
+
+    fn line(&self, elapsed: Duration) -> String {
+        progress_line(&self.label, self.done, self.total, elapsed)
+    }
+}
+
+/// `export: Email 1,200/5,000 (24%), 40/s, about 1m35s left`. The rate and
+/// the time left are left out until there is enough to estimate them from.
+pub fn progress_line(label: &str, done: u64, total: u64, elapsed: Duration) -> String {
+    let pct = (done * 100).checked_div(total).unwrap_or(100);
+    let mut out = format!("{label} {}/{} ({pct}%)", thousands(done), thousands(total));
+    let secs = elapsed.as_secs_f64();
+    if done > 0 && secs >= 1.0 {
+        let rate = done as f64 / secs;
+        out.push_str(&format!(", {}/s", format_rate(rate)));
+        let left = (total - done) as f64 / rate;
+        if total > done && left.is_finite() {
+            out.push_str(&format!(
+                ", about {} left",
+                format_duration(Duration::from_secs_f64(left))
+            ));
+        }
+    }
+    out
+}
+
+/// A duration as `45s`, `1m35s` or `2h03m`.
+pub fn format_duration(d: Duration) -> String {
+    let total = d.as_secs();
+    if total < 60 {
+        format!("{total}s")
+    } else if total < 3600 {
+        format!("{}m{:02}s", total / 60, total % 60)
+    } else {
+        format!("{}h{:02}m", total / 3600, (total % 3600) / 60)
+    }
+}
+
+/// The line printed when a type is finished: `export: Email done: 120
+/// created, 3 updated, 5,000 unchanged, 0 failed (2m03s)`.
+pub fn done_line(label: &str, counts: &crate::sync::TypeCounts, elapsed: Duration) -> String {
+    format!(
+        "{label} done: {} created, {} updated, {} unchanged, {} failed ({})",
+        thousands(counts.created),
+        thousands(counts.updated),
+        thousands(counts.skipped),
+        thousands(counts.failed),
+        format_duration(elapsed)
+    )
+}
+
+fn format_rate(rate: f64) -> String {
+    if rate >= 10.0 {
+        format!("{:.0}", rate)
+    } else {
+        format!("{:.1}", rate)
+    }
+}
+
+pub fn thousands(n: u64) -> String {
+    let digits = n.to_string();
+    let mut out = String::with_capacity(digits.len() + digits.len() / 3);
+    for (i, c) in digits.chars().enumerate() {
+        if i > 0 && (digits.len() - i).is_multiple_of(3) {
+            out.push(',');
+        }
+        out.push(c);
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_line_has_count_rate_and_time_left() {
+        let line = progress_line("export: Email", 1200, 5000, Duration::from_secs(30));
+        assert!(
+            line.starts_with("export: Email 1,200/5,000 (24%)"),
+            "{line}"
+        );
+        assert!(line.contains("40/s"), "{line}");
+        assert!(line.contains("about 1m35s left"), "{line}");
+    }
+
+    #[test]
+    fn no_estimate_before_there_is_something_to_estimate_from() {
+        assert_eq!(
+            progress_line("export: Email", 0, 10, Duration::from_secs(5)),
+            "export: Email 0/10 (0%)"
+        );
+        assert_eq!(
+            progress_line("export: Email", 3, 10, Duration::from_millis(200)),
+            "export: Email 3/10 (30%)"
+        );
+    }
+
+    #[test]
+    fn a_finished_run_shows_no_time_left() {
+        let line = progress_line("export: Email", 10, 10, Duration::from_secs(4));
+        assert!(!line.contains("left"), "{line}");
+        assert!(line.contains("(100%)"), "{line}");
+    }
+
+    #[test]
+    fn durations_and_counts_read_naturally() {
+        assert_eq!(format_duration(Duration::from_secs(45)), "45s");
+        assert_eq!(format_duration(Duration::from_secs(95)), "1m35s");
+        assert_eq!(format_duration(Duration::from_secs(7380)), "2h03m");
+        assert_eq!(thousands(0), "0");
+        assert_eq!(thousands(999), "999");
+        assert_eq!(thousands(1_000), "1,000");
+        assert_eq!(thousands(1_234_567), "1,234,567");
+    }
+
+    #[test]
+    fn the_done_line_names_every_count() {
+        let counts = crate::sync::TypeCounts {
+            created: 120,
+            updated: 3,
+            skipped: 5000,
+            failed: 1,
+            ..Default::default()
+        };
+        assert_eq!(
+            done_line("export: Email", &counts, Duration::from_secs(123)),
+            "export: Email done: 120 created, 3 updated, 5,000 unchanged, 1 failed (2m03s)"
+        );
+    }
+
+    #[test]
+    fn a_quiet_logger_or_empty_job_prints_nothing() {
+        let p = Progress::new("x", 10, &Logger::from_flags(true, 0));
+        assert!(!p.enabled);
+        let p = Progress::new("x", 0, &Logger::from_flags(false, 0));
+        assert!(!p.enabled);
+    }
+}

tests/integration_cyrus.rs

@@ -38,6 +38,7 @@ fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig {
         automap: true,
         include_deleted: false,
         fetch_batch: 64,
+        fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
         imap_connections: 2,
         allow_source_change: false,
     }

tests/integration_dovecot.rs

@@ -43,6 +43,7 @@ fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportCon
         automap: true,
         include_deleted: false,
         fetch_batch: 64,
+        fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
         imap_connections: 2,
         allow_source_change: false,
     }

tests/mock_exchange_ews.rs

@@ -864,6 +864,7 @@ fn for_each_fetched_item_streams_every_id_across_windows() {
         url: &url,
         source_id: 1,
         batch_size: 1,
+        batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
         attachment_batch: 1,
         connections: 2,
         use_syncfolderitems: false,
@@ -873,7 +874,8 @@ fn for_each_fetched_item_streams_every_id_across_windows() {
     let ids: Vec<ItemId> = (0..5).map(|i| ItemId::new(format!("I{i}"), "K")).collect();
 
     let mut delivered = 0usize;
-    let failed = for_each_fetched_item(&ctx, ItemShape::Message, &ids, |msg| {
+    let failed =
+        for_each_fetched_item(&ctx, ItemShape::Message, &ids, &Default::default(), |msg| {
             assert!(msg.success);
             delivered += 1;
             Ok(())
@@ -885,6 +887,73 @@ fn for_each_fetched_item_streams_every_id_across_windows() {
     _m.assert();
 }
 
+#[test]
+fn getitem_batches_are_split_by_bytes_when_sizes_are_known() {
+    use inbuxa_migrate::logging::Logger;
+    use inbuxa_migrate::sync::import_exchange_ews::items::{ItemRunCtx, for_each_fetched_item};
+    use std::collections::HashMap;
+
+    let mut server = mockito::Server::new();
+    let url = format!("{}/EWS/Exchange.asmx", server.url());
+    let one_message = envelope(&format!(
+        "<m:GetItemResponse{NS}><m:ResponseMessages><m:GetItemResponseMessage ResponseClass=\"Success\">\
+         <m:ResponseCode>NoError</m:ResponseCode><m:Items><t:Message><t:ItemId Id=\"X\" ChangeKey=\"K\"/></t:Message></m:Items>\
+         </m:GetItemResponseMessage></m:ResponseMessages></m:GetItemResponse>"
+    ));
+    // Ten items fit one batch by count, but at 20 bytes each and a 30-byte
+    // cap every item goes alone: four GetItem calls, not one.
+    let m = server
+        .mock("POST", "/EWS/Exchange.asmx")
+        .with_status(200)
+        .with_header("content-type", TXT_XML)
+        .with_body(&one_message)
+        .expect(4)
+        .create();
+    let c = client(0);
+    let ctx = ItemRunCtx {
+        client: &c,
+        url: &url,
+        source_id: 1,
+        batch_size: 10,
+        batch_bytes: 30,
+        attachment_batch: 1,
+        connections: 2,
+        use_syncfolderitems: false,
+        sync_batch: 512,
+        logger: Logger::new(0),
+    };
+    let ids: Vec<ItemId> = (0..4).map(|i| ItemId::new(format!("I{i}"), "K")).collect();
+    let sizes: HashMap<String, u64> = ids.iter().map(|id| (id.id.clone(), 20)).collect();
+    let mut delivered = 0usize;
+    for_each_fetched_item(&ctx, ItemShape::Message, &ids, &sizes, |_| {
+        delivered += 1;
+        Ok(())
+    })
+    .expect("fetch");
+    assert_eq!(delivered, 4);
+    m.assert();
+}
+
+#[test]
+fn find_item_reports_each_items_size() {
+    use inbuxa_migrate::exchange_ews::parse::parse_find_item_response;
+    let body = envelope(&format!(
+        "<m:FindItemResponse{NS}><m:ResponseMessages><m:FindItemResponseMessage ResponseClass=\"Success\">\
+         <m:ResponseCode>NoError</m:ResponseCode>\
+         <m:RootFolder TotalItemsInView=\"2\" IncludesLastItemInRange=\"true\"><t:Items>\
+         <t:Message><t:ItemId Id=\"A\" ChangeKey=\"K\"/><t:Size>1234</t:Size></t:Message>\
+         <t:Message><t:ItemId Id=\"B\" ChangeKey=\"K\"/></t:Message>\
+         </t:Items></m:RootFolder></m:FindItemResponseMessage></m:ResponseMessages></m:FindItemResponse>"
+    ));
+    let r = parse_find_item_response(body.as_bytes()).unwrap();
+    assert_eq!(r.items.len(), 2);
+    assert_eq!(
+        (r.items[0].id.id.as_str(), r.items[0].size),
+        ("A", Some(1234))
+    );
+    assert_eq!((r.items[1].id.id.as_str(), r.items[1].size), ("B", None));
+}
+
 #[test]
 fn warning_response_class_is_treated_as_success_in_mock() {
     let body = envelope(&format!(

tests/mock_export_batch.rs

@@ -0,0 +1,342 @@
+/*
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+//! Batched `Email/import` on export: batches honor the server's limits, one
+//! rejected message fails alone, and a batch that ends without a clear answer
+//! is settled against the target instead of being sent again.
+
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use inbuxa_migrate::db;
+use inbuxa_migrate::jmap::account::AccountSelector;
+use inbuxa_migrate::jmap::http::Auth;
+use inbuxa_migrate::logging::Logger;
+use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, TypeCounts};
+use mockito::Matcher;
+use serde_json::{Value, json};
+
+const API: &str = "/jmap/api";
+
+fn tmp() -> PathBuf {
+    static SEQ: AtomicUsize = AtomicUsize::new(0);
+    let n = SEQ.fetch_add(1, Ordering::Relaxed);
+    let mut p = std::env::temp_dir();
+    p.push(format!(
+        "inbuxa-migrate-exportbatch-{}-{:?}-{n}.sqlite",
+        std::process::id(),
+        std::thread::current().id(),
+    ));
+    let _ = std::fs::remove_file(&p);
+    p
+}
+
+fn session(base: &str, max_objects_in_set: u64, max_concurrent_upload: u64) -> String {
+    json!({
+        "apiUrl": format!("{base}{API}"),
+        "uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
+        "downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
+        "capabilities": { "urn:ietf:params:jmap:core": {
+            "maxObjectsInGet": 500, "maxObjectsInSet": max_objects_in_set,
+            "maxCallsInRequest": 16, "maxConcurrentRequests": 4,
+            "maxConcurrentUpload": max_concurrent_upload,
+            "maxSizeRequest": 10000000, "maxSizeUpload": 50000000
+        } },
+        "accounts": { "w": { "name": "alice",
+            "accountCapabilities": { "urn:ietf:params:jmap:mail": {} } } }
+    })
+    .to_string()
+}
+
+/// An archive with one Inbox and `n` distinct messages, `<m-1@h>` .. `<m-n@h>`.
+fn seed(n: usize) -> PathBuf {
+    let archive = tmp();
+    let conn = db::init::open(&archive).unwrap();
+    conn.execute(
+        "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
+        [],
+    )
+    .unwrap();
+    for i in 1..=n {
+        let raw = format!("From: a@x\r\nSubject: m{i}\r\nMessage-ID: <m-{i}@h>\r\n\r\nbody {i}");
+        let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
+        let mm = inbuxa_migrate::sync::keys::index_to_json(
+            &inbuxa_migrate::sync::emailmeta::email_index_from_blob(raw.as_bytes()),
+        );
+        conn.execute(
+            "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match)
+             VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]',?2)",
+            rusqlite::params![blob, mm],
+        )
+        .unwrap();
+    }
+    archive
+}
+
+/// The session, an Inbox already on the target, and uploads. The target's
+/// email list is left to each test.
+fn mock_target(server: &mut mockito::ServerGuard, session_body: String) -> Vec<mockito::Mock> {
+    vec![
+        server.mock("GET", "/").with_status(404).create(),
+        server
+            .mock("GET", "/.well-known/jmap")
+            .with_body(session_body)
+            .create(),
+        server
+            .mock("POST", API)
+            .match_body(Matcher::Regex("Mailbox/query".into()))
+            .with_body(
+                json!({"methodResponses":[["Mailbox/query",
+                    {"accountId":"w","ids":["t1"]},"q"]]})
+                .to_string(),
+            )
+            .create(),
+        server
+            .mock("POST", API)
+            .match_body(Matcher::AllOf(vec![
+                Matcher::Regex("Mailbox/query".into()),
+                Matcher::Regex("anchor".into()),
+            ]))
+            .with_body(
+                json!({"methodResponses":[["Mailbox/query",
+                    {"accountId":"w","ids":[]},"q"]]})
+                .to_string(),
+            )
+            .create(),
+        server
+            .mock("POST", API)
+            .match_body(Matcher::Regex("Mailbox/get".into()))
+            .with_body(
+                json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
+                    {"id":"t1","name":"Inbox","role":"inbox","parentId":null,
+                     "myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
+                .to_string(),
+            )
+            .create(),
+        server
+            .mock("POST", Matcher::Regex("/jmap/upload/".into()))
+            .with_body(json!({"blobId":"UP"}).to_string())
+            .create(),
+    ]
+}
+
+fn empty_email_query(server: &mut mockito::ServerGuard) -> mockito::Mock {
+    server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/query".into()))
+        .with_body(
+            json!({"methodResponses":[["Email/query",{"accountId":"w","ids":[]},"q"]]}).to_string(),
+        )
+        .create()
+}
+
+/// The creation ids of the emails in an `Email/import` request body.
+fn import_cids(body: &[u8]) -> Vec<String> {
+    let v: Value = serde_json::from_slice(body).unwrap_or(Value::Null);
+    v["methodCalls"][0][1]["emails"]
+        .as_object()
+        .map(|m| m.keys().cloned().collect())
+        .unwrap_or_default()
+}
+
+/// An `Email/import` answer: each id in `cids` created, except those in
+/// `rejected`, which come back as `invalidEmail`.
+fn import_answer(cids: &[String], rejected: &[&str]) -> Vec<u8> {
+    let mut created = serde_json::Map::new();
+    let mut not_created = serde_json::Map::new();
+    for cid in cids {
+        if rejected.contains(&cid.as_str()) {
+            not_created.insert(cid.clone(), json!({"type":"invalidEmail"}));
+        } else {
+            created.insert(
+                cid.clone(),
+                json!({"id": format!("T{cid}"), "blobId":"b","threadId":"t","size":10}),
+            );
+        }
+    }
+    json!({"methodResponses":[["Email/import",
+        {"accountId":"w","created":created,"notCreated":not_created},"i"]]})
+    .to_string()
+    .into_bytes()
+}
+
+fn run_export(archive: &Path, base: &str, threads: usize) -> TypeCounts {
+    let summary = sync::export::run(
+        CommonConfig {
+            archive: archive.to_path_buf(),
+            threads,
+            dry_run: false,
+            max_retries: 1,
+            allow_invalid_certs: false,
+            logger: Logger::from_flags(true, 0),
+        },
+        ExportConfig {
+            connect: ConnectConfig {
+                url: base.to_owned(),
+                auth: Auth::Basic {
+                    user: "u".into(),
+                    password: "p".into(),
+                },
+                account: AccountSelector::Id("w".into()),
+            },
+            objects: None,
+            prune: false,
+            yes: true,
+        },
+    )
+    .expect("export run");
+    summary
+        .per_type
+        .iter()
+        .find(|(t, _)| *t == "Email")
+        .map(|(_, c)| c.clone())
+        .expect("email counts")
+}
+
+#[test]
+fn imports_are_batched_up_to_max_objects_in_set() {
+    let mut server = mockito::Server::new();
+    let base = server.url();
+    let archive = seed(5);
+    let _base_mocks = mock_target(&mut server, session(&base, 2, 4));
+    let _eq = empty_email_query(&mut server);
+
+    let sizes = Arc::new(std::sync::Mutex::new(Vec::new()));
+    let seen = sizes.clone();
+    let imports = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/import".into()))
+        .with_body_from_request(move |req| {
+            let cids = import_cids(req.body().unwrap());
+            seen.lock().unwrap().push(cids.len());
+            import_answer(&cids, &[])
+        })
+        .expect(3)
+        .create();
+
+    let email = run_export(&archive, &base, 4);
+    assert_eq!(email.created, 5);
+    assert_eq!(email.failed, 0);
+    imports.assert();
+    let mut sizes = sizes.lock().unwrap().clone();
+    sizes.sort_unstable();
+    assert_eq!(
+        sizes,
+        vec![1, 2, 2],
+        "no call carries more than maxObjectsInSet"
+    );
+    let _ = std::fs::remove_file(&archive);
+}
+
+#[test]
+fn a_rejected_message_in_a_batch_fails_alone() {
+    let mut server = mockito::Server::new();
+    let base = server.url();
+    let archive = seed(3);
+    let _base_mocks = mock_target(&mut server, session(&base, 50, 4));
+    let _eq = empty_email_query(&mut server);
+
+    let rejected = Arc::new(std::sync::Mutex::new(String::new()));
+    let pick = rejected.clone();
+    let imports = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/import".into()))
+        .with_body_from_request(move |req| {
+            let cids = import_cids(req.body().unwrap());
+            let mut sorted = cids.clone();
+            sorted.sort();
+            let middle = sorted[1].clone();
+            *pick.lock().unwrap() = middle.clone();
+            import_answer(&cids, &[middle.as_str()])
+        })
+        .expect(1)
+        .create();
+
+    let email = run_export(&archive, &base, 1);
+    assert_eq!(email.created, 2, "the other two land");
+    assert_eq!(email.failed, 1, "only {} fails", rejected.lock().unwrap());
+    imports.assert();
+    let _ = std::fs::remove_file(&archive);
+}
+
+#[test]
+fn an_unclear_batch_is_settled_against_the_target_not_resent() {
+    let mut server = mockito::Server::new();
+    let base = server.url();
+    let archive = seed(2);
+    let _base_mocks = mock_target(&mut server, session(&base, 50, 4));
+
+    // First look: the target is empty. After the unclear batch: m-1 arrived.
+    let queries = Arc::new(AtomicUsize::new(0));
+    let q = queries.clone();
+    let _eq = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/query".into()))
+        .with_body_from_request(move |req| {
+            // A page after the first (it carries an anchor) is empty.
+            let paged = String::from_utf8_lossy(req.body().unwrap()).contains("anchor");
+            let ids = if paged || q.fetch_add(1, Ordering::SeqCst) == 0 {
+                json!([])
+            } else {
+                json!(["arrived"])
+            };
+            json!({"methodResponses":[["Email/query",{"accountId":"w","ids":ids},"q"]]})
+                .to_string()
+                .into_bytes()
+        })
+        .create();
+    let _eg = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/get".into()))
+        .with_body(
+            json!({"methodResponses":[["Email/get",{"accountId":"w","list":[
+                {"id":"arrived","messageId":["m-1@h"],"mailboxIds":{"t1":true},"keywords":{}}
+            ],"notFound":[]},"g"]]})
+            .to_string(),
+        )
+        .create();
+
+    // The batch carries both messages and gets a gateway timeout, so it may
+    // or may not have been applied. mockito answers with the first matching
+    // mock still owed hits, so this answers the first import only; every
+    // later import is created by the next mock, which records what it sees.
+    let gateway = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/import".into()))
+        .with_status(504)
+        .expect(1)
+        .create();
+    let later = Arc::new(std::sync::Mutex::new(Vec::new()));
+    let later_seen = later.clone();
+    let _created = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("Email/import".into()))
+        .with_body_from_request(move |req| {
+            let cids = import_cids(req.body().unwrap());
+            later_seen.lock().unwrap().push(cids.clone());
+            import_answer(&cids, &[])
+        })
+        .create();
+
+    let email = run_export(&archive, &base, 1);
+    gateway.assert();
+    assert!(
+        queries.load(Ordering::SeqCst) >= 2,
+        "the target is read again before anything is resent"
+    );
+    assert_eq!(
+        *later.lock().unwrap(),
+        vec![vec!["e2".to_owned()]],
+        "only the message that did not arrive is sent again, on its own"
+    );
+    assert_eq!(
+        email.created, 2,
+        "one found on the target, one imported again"
+    );
+    assert_eq!(email.failed, 0);
+    let _ = std::fs::remove_file(&archive);
+}

tests/mock_export_dry_run.rs

@@ -0,0 +1,325 @@
+/*
+ * SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+//! `export --dry-run` predicts what a real run would fail on: a message
+//! larger than the target accepts, an object too big for one request, and a
+//! Sieve script the target rejects. It keeps the counts, so the run exits
+//! non-zero just as the real one would, and it writes nothing.
+
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use inbuxa_migrate::db;
+use inbuxa_migrate::jmap::account::AccountSelector;
+use inbuxa_migrate::jmap::http::Auth;
+use inbuxa_migrate::logging::Logger;
+use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, Summary, TypeCounts};
+use mockito::Matcher;
+use serde_json::json;
+
+const API: &str = "/jmap/api";
+
+fn tmp() -> PathBuf {
+    static SEQ: AtomicUsize = AtomicUsize::new(0);
+    let n = SEQ.fetch_add(1, Ordering::Relaxed);
+    let mut p = std::env::temp_dir();
+    p.push(format!(
+        "inbuxa-migrate-exportdry-{}-{:?}-{n}.sqlite",
+        std::process::id(),
+        std::thread::current().id(),
+    ));
+    let _ = std::fs::remove_file(&p);
+    p
+}
+
+fn session(base: &str, max_size_upload: u64, max_size_request: u64, sieve: &[&str]) -> String {
+    json!({
+        "apiUrl": format!("{base}{API}"),
+        "uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
+        "downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
+        "capabilities": { "urn:ietf:params:jmap:core": {
+            "maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16,
+            "maxConcurrentRequests": 4, "maxConcurrentUpload": 4,
+            "maxSizeRequest": max_size_request, "maxSizeUpload": max_size_upload
+        } },
+        "accounts": { "w": { "name": "alice",
+            "accountCapabilities": {
+                "urn:ietf:params:jmap:mail": {},
+                "urn:ietf:params:jmap:contacts": {},
+                "urn:ietf:params:jmap:sieve": { "sieveExtensions": sieve }
+            } } }
+    })
+    .to_string()
+}
+
+fn dry_run(archive: &Path, base: &str) -> Summary {
+    sync::export::run(
+        CommonConfig {
+            archive: archive.to_path_buf(),
+            threads: 2,
+            dry_run: true,
+            max_retries: 0,
+            allow_invalid_certs: false,
+            logger: Logger::from_flags(true, 0),
+        },
+        ExportConfig {
+            connect: ConnectConfig {
+                url: base.to_owned(),
+                auth: Auth::Basic {
+                    user: "u".into(),
+                    password: "p".into(),
+                },
+                account: AccountSelector::Id("w".into()),
+            },
+            objects: None,
+            prune: false,
+            yes: true,
+        },
+    )
+    .expect("a dry run returns its plan")
+}
+
+fn counts(summary: &Summary, ty: &str) -> TypeCounts {
+    summary
+        .per_type
+        .iter()
+        .find(|(t, _)| *t == ty)
+        .map(|(_, c)| c.clone())
+        .unwrap_or_else(|| panic!("no counts for {ty}: {summary:?}"))
+}
+
+fn root_and_session(server: &mut mockito::ServerGuard, body: String) -> Vec<mockito::Mock> {
+    vec![
+        server.mock("GET", "/").with_status(404).create(),
+        server
+            .mock("GET", "/.well-known/jmap")
+            .with_body(body)
+            .create(),
+    ]
+}
+
+/// Nothing that writes to the account: no `/set`, no import. Returns the
+/// mocks, each expecting no calls.
+fn no_writes(server: &mut mockito::ServerGuard) -> Vec<mockito::Mock> {
+    ["/set\"", "Email/import"]
+        .into_iter()
+        .map(|m| {
+            server
+                .mock("POST", API)
+                .match_body(Matcher::Regex(m.into()))
+                .expect(0)
+                .create()
+        })
+        .collect()
+}
+
+fn empty(
+    server: &mut mockito::ServerGuard,
+    method: &str,
+    reply: serde_json::Value,
+) -> mockito::Mock {
+    server
+        .mock("POST", API)
+        .match_body(Matcher::Regex(method.into()))
+        .with_body(json!({ "methodResponses": [[method, reply, "x"]] }).to_string())
+        .create()
+}
+
+#[test]
+fn a_message_too_large_to_upload_is_predicted_to_fail() {
+    let mut server = mockito::Server::new();
+    let base = server.url();
+    let archive = tmp();
+    {
+        let conn = db::init::open(&archive).unwrap();
+        conn.execute(
+            "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
+            [],
+        )
+        .unwrap();
+        for body in ["short".to_owned(), "x".repeat(4000)] {
+            let raw = format!(
+                "From: a@x\r\nSubject: s\r\nMessage-ID: <{}@h>\r\n\r\n{body}",
+                body.len()
+            );
+            let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
+            conn.execute(
+                "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
+                 VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')",
+                rusqlite::params![blob],
+            )
+            .unwrap();
+        }
+    }
+    let _s = root_and_session(&mut server, session(&base, 1000, 10_000_000, &[]));
+    let _mq = empty(
+        &mut server,
+        "Mailbox/query",
+        json!({"accountId":"w","ids":[]}),
+    );
+    let _eq = empty(
+        &mut server,
+        "Email/query",
+        json!({"accountId":"w","ids":[]}),
+    );
+    let no_upload = server
+        .mock("POST", Matcher::Regex("/jmap/upload/".into()))
+        .expect(0)
+        .create();
+    let writes = no_writes(&mut server);
+
+    let summary = dry_run(&archive, &base);
+    let email = counts(&summary, "Email");
+    assert_eq!(email.created, 1, "the short one would be created");
+    assert_eq!(email.failed, 1, "the long one is over maxSizeUpload");
+    assert!(
+        summary.any_failed(),
+        "so the dry run exits non-zero, like a real run"
+    );
+    no_upload.assert();
+    for w in writes {
+        w.assert();
+    }
+    let _ = std::fs::remove_file(&archive);
+}
+
+#[test]
+fn a_contact_too_large_for_one_request_is_predicted_to_fail() {
+    let mut server = mockito::Server::new();
+    let base = server.url();
+    let archive = tmp();
+    {
+        let conn = db::init::open(&archive).unwrap();
+        conn.execute(
+            "INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
+            [],
+        )
+        .unwrap();
+        let photo = db::blobs::intern_blob(&conn, &vec![b'P'; 8000]).unwrap();
+        let huge = json!({ "@type": "Card", "name": { "full": "Photo Person" },
+            "media": { "photo": { "@type": "Media", "kind": "photo",
+                "@blob": photo, "mediaType": "image/png" } } })
+        .to_string();
+        let small = json!({ "@type": "Card", "name": { "full": "Small Person" } }).to_string();
+        for (id, uid, data) in [(1, "huge-card", &huge), (2, "small-card", &small)] {
+            conn.execute(
+                "INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (?1,?2,'[1]',?3)",
+                rusqlite::params![id, uid, data],
+            )
+            .unwrap();
+        }
+    }
+    let _s = root_and_session(&mut server, session(&base, 50_000_000, 4000, &[]));
+    let _ab = empty(
+        &mut server,
+        "AddressBook/get",
+        json!({"accountId":"w","list":[],"notFound":[]}),
+    );
+    let _cq = empty(
+        &mut server,
+        "ContactCard/query",
+        json!({"accountId":"w","ids":[]}),
+    );
+    let writes = no_writes(&mut server);
+
+    let summary = dry_run(&archive, &base);
+    let cards = counts(&summary, "ContactCard");
+    assert_eq!(cards.created, 1, "the small card would be created");
+    assert_eq!(
+        cards.failed, 1,
+        "the card with the photo inlined is over maxSizeRequest"
+    );
+    assert!(summary.any_failed());
+    for w in writes {
+        w.assert();
+    }
+    let _ = std::fs::remove_file(&archive);
+}
+
+/// One active script with Stalwart's `vnd.stalwart.while`, against a target
+/// that names it `vnd.inbuxa.while`; `validate` is the answer to
+/// `SieveScript/validate`. Checks that the script sent for validation is the
+/// renamed one, and returns the run's counts.
+fn dry_run_one_script(validate: serde_json::Value) -> Summary {
+    let mut server = mockito::Server::new();
+    let base = server.url();
+    let archive = tmp();
+    {
+        let conn = db::init::open(&archive).unwrap();
+        let blob = db::blobs::intern_blob(
+            &conn,
+            b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
+        )
+        .unwrap();
+        conn.execute(
+            "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'main',1,?1)",
+            rusqlite::params![blob],
+        )
+        .unwrap();
+    }
+    let _s = root_and_session(
+        &mut server,
+        session(
+            &base,
+            50_000_000,
+            10_000_000,
+            &["fileinto", "vnd.inbuxa.while"],
+        ),
+    );
+    let _get = empty(
+        &mut server,
+        "SieveScript/get",
+        json!({"accountId":"w","list":[],"notFound":[]}),
+    );
+    let upload = server
+        .mock("POST", Matcher::Regex("/jmap/upload/".into()))
+        .match_body(Matcher::Regex("vnd\\.inbuxa\\.while".into()))
+        .with_body(json!({"blobId":"TMP"}).to_string())
+        .expect(1)
+        .create();
+    let _validate = server
+        .mock("POST", API)
+        .match_body(Matcher::Regex("SieveScript/validate".into()))
+        .with_body(json!({"methodResponses":[validate]}).to_string())
+        .create();
+    let writes = no_writes(&mut server);
+    let summary = dry_run(&archive, &base);
+    for w in writes {
+        w.assert();
+    }
+    upload.assert();
+    let _ = std::fs::remove_file(&archive);
+    summary
+}
+
+#[test]
+fn a_sieve_script_the_target_rejects_is_predicted_to_fail() {
+    let summary = dry_run_one_script(json!(["SieveScript/validate",
+        {"accountId":"w","error":{"type":"invalidScript","description":"unknown test"}},"v"]));
+    let sieve = counts(&summary, "SieveScript");
+    assert_eq!(sieve.failed, 1);
+    assert_eq!(sieve.created, 0);
+    assert!(summary.any_failed());
+}
+
+#[test]
+fn a_valid_sieve_script_is_validated_after_renaming_and_nothing_is_written() {
+    let summary = dry_run_one_script(json!(["SieveScript/validate",
+        {"accountId":"w","error":null},"v"]));
+    let sieve = counts(&summary, "SieveScript");
+    assert_eq!(sieve.created, 1, "it would be created");
+    assert_eq!(sieve.failed, 0);
+    assert!(!summary.any_failed());
+}
+
+#[test]
+fn a_target_without_validate_still_gets_a_plan() {
+    let summary = dry_run_one_script(json!(["error",
+        {"type":"unknownMethod"},"v"]));
+    let sieve = counts(&summary, "SieveScript");
+    assert_eq!(sieve.created, 1, "not checked, so planned as written");
+    assert_eq!(sieve.failed, 0);
+}

tests/mock_imap.rs

@@ -169,6 +169,7 @@ fn run_import(
         automap: true,
         include_deleted: false,
         fetch_batch: 256,
+        fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
         imap_connections: 1,
         allow_source_change: false,
     };
@@ -2005,3 +2006,218 @@ fn assert_name_selected_as_listed(listed: &'static str, stored: &str, archive_na
     );
     let _ = std::fs::remove_file(&archive);
 }
+
+fn write_fetch_message_dated(
+    conn: &mut MockConn,
+    seq: u32,
+    uid: u32,
+    internaldate: &str,
+    body: &[u8],
+) -> std::io::Result<()> {
+    let header = format!(
+        "* {seq} FETCH (UID {uid} FLAGS (\\Seen) INTERNALDATE \"{internaldate}\" RFC822.SIZE {} BODY[] {{{}}}\r\n",
+        body.len(),
+        body.len()
+    );
+    conn.write_raw(header.as_bytes())?;
+    conn.write_raw(body)?;
+    conn.write_raw(b")\r\n")
+}
+
+const BODY_A: &[u8] = b"From: a@b\r\nMessage-ID: <a@h>\r\nSubject: a\r\n\r\none";
+const BODY_B: &[u8] = b"From: a@b\r\nMessage-ID: <b@h>\r\nSubject: b\r\n\r\ntwo";
+const BODY_C: &[u8] = b"From: a@b\r\nMessage-ID: <c@h>\r\nSubject: c\r\n\r\nthree";
+
+fn email_counts(summary: &inbuxa_migrate::sync::Summary) -> inbuxa_migrate::sync::TypeCounts {
+    summary
+        .per_type
+        .iter()
+        .find(|(k, _)| *k == "email")
+        .map(|(_, c)| c.clone())
+        .expect("email counts")
+}
+
+#[test]
+fn a_message_that_will_not_import_is_recorded_and_the_folder_carries_on() {
+    let worker: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
+        auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
+        let (tag, _) = conn.read_command()?;
+        write_select(conn, &tag, 12345, 4, 3)?;
+        let (tag, cmd) = conn.read_command()?;
+        assert!(cmd.starts_with("UID FETCH"), "got {cmd}");
+        write_fetch_message_dated(conn, 1, 1, "12-May-2025 10:00:00 +0000", BODY_A)?;
+        // No such month: this one cannot be imported.
+        write_fetch_message_dated(conn, 2, 2, "12-Mai-2025 10:00:00 +0000", BODY_B)?;
+        // Lower case is only untidy, and is imported.
+        write_fetch_message_dated(conn, 3, 3, "12-may-2025 10:00:00 +0000", BODY_C)?;
+        conn.write_line(&format!("{tag} OK"))?;
+        drain_until_close(conn);
+        Ok(())
+    });
+    let server = MockImap::start_scripts(vec![
+        control_script_one_folder(12345, 4, &[1, 2, 3]),
+        worker,
+    ]);
+    let archive = tempfile("bad-message");
+    let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("import");
+    let email = email_counts(&summary);
+    assert_eq!(email.created, 2, "summary={summary:?}");
+    assert_eq!(email.failed, 1, "summary={summary:?}");
+    let conn = Connection::open(&archive).unwrap();
+    db::init::apply_schema(&conn).unwrap();
+    assert_eq!(count(&conn, "emails"), 2);
+    let uids: Vec<i64> = conn
+        .prepare("SELECT uid FROM sync_id_imap WHERE type_name = 'email' ORDER BY uid")
+        .unwrap()
+        .query_map([], |r| r.get(0))
+        .unwrap()
+        .collect::<Result<_, _>>()
+        .unwrap();
+    assert_eq!(
+        uids,
+        vec![1, 3],
+        "the failed message must stay out of the UID map"
+    );
+}
+
+#[test]
+fn a_failed_chunk_keeps_the_ones_before_it_and_a_rerun_fetches_only_what_is_missing() {
+    // The archive is opened with an exclusive lock, so the commit after each
+    // chunk cannot be watched from outside while a run is going; what can be
+    // checked is its effect. First run, one UID per chunk: chunk 1 arrives,
+    // chunk 2 fails the way a dying server would.
+    let archive = tempfile("chunk-commit");
+    let worker_1: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
+        auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
+        let (tag, _) = conn.read_command()?;
+        write_select(conn, &tag, 777, 3, 2)?;
+        let (tag, cmd) = conn.read_command()?;
+        assert!(cmd.starts_with("UID FETCH 1 "), "got {cmd}");
+        write_fetch_message(conn, 1, 1, BODY_A)?;
+        conn.write_line(&format!("{tag} OK"))?;
+        let (tag, cmd) = conn.read_command()?;
+        assert!(cmd.starts_with("UID FETCH 2 "), "got {cmd}");
+        conn.write_line(&format!("{tag} NO [SERVERBUG] gone"))?;
+        drain_until_close(conn);
+        Ok(())
+    });
+    // Second run: only UID 2 is missing, so only UID 2 may be fetched.
+    let worker_2: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
+        auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
+        let (tag, _) = conn.read_command()?;
+        write_select(conn, &tag, 777, 3, 2)?;
+        let (tag, cmd) = conn.read_command()?;
+        assert!(
+            cmd.starts_with("UID FETCH 2 "),
+            "rerun refetched more than UID 2: {cmd}"
+        );
+        write_fetch_message(conn, 2, 2, BODY_B)?;
+        conn.write_line(&format!("{tag} OK"))?;
+        drain_until_close(conn);
+        Ok(())
+    });
+    let server = MockImap::start_scripts(vec![
+        control_script_one_folder(777, 3, &[1, 2]),
+        worker_1,
+        control_script_one_folder(777, 3, &[1, 2]),
+        worker_2,
+    ]);
+
+    let first =
+        run_import(&server, "alice", archive.clone(), |c| c.fetch_batch = 1).expect("first run");
+    let e1 = email_counts(&first);
+    assert_eq!((e1.created, e1.failed), (1, 1), "first={first:?}");
+
+    let second =
+        run_import(&server, "alice", archive.clone(), |c| c.fetch_batch = 1).expect("second run");
+    let e2 = email_counts(&second);
+    assert_eq!((e2.created, e2.failed), (1, 0), "second={second:?}");
+    let conn = Connection::open(&archive).unwrap();
+    db::init::apply_schema(&conn).unwrap();
+    assert_eq!(count(&conn, "emails"), 2);
+}
+
+#[test]
+fn body_fetches_are_split_by_message_size() {
+    // Three 20-byte messages under a 30-byte cap: the control connection
+    // learns the sizes first, and each body is then fetched on its own.
+    let control: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
+        auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
+        let (tag, _) = conn.read_command()?;
+        conn.write_line("* LIST () \"/\" \"INBOX\"")?;
+        conn.write_line(&format!("{tag} OK"))?;
+        let (tag, _) = conn.read_command()?;
+        conn.write_line(&format!("{tag} OK"))?;
+        let (tag, _) = conn.read_command()?;
+        write_select(conn, &tag, 555, 4, 3)?;
+        let (tag, _) = conn.read_command()?;
+        conn.write_line("* SEARCH 1 2 3")?;
+        conn.write_line(&format!("{tag} OK"))?;
+        let (tag, cmd) = conn.read_command()?;
+        assert_eq!(cmd, "UID FETCH 1:3 (UID RFC822.SIZE)");
+        for uid in 1..=3 {
+            conn.write_line(&format!("* {uid} FETCH (UID {uid} RFC822.SIZE 20)"))?;
+        }
+        conn.write_line(&format!("{tag} OK"))?;
+        drain_until_close(conn);
+        Ok(())
+    });
+    let fetches = std::sync::Arc::new(Mutex::new(Vec::<String>::new()));
+    let worker: Script = {
+        let fetches = fetches.clone();
+        Box::new(move |conn: &mut MockConn| -> std::io::Result<()> {
+            auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
+            let (tag, _) = conn.read_command()?;
+            write_select(conn, &tag, 555, 4, 3)?;
+            let bodies: [&[u8]; 3] = [BODY_A, BODY_B, BODY_C];
+            for _ in 0..3 {
+                let (tag, cmd) = conn.read_command()?;
+                let uid: u32 = cmd
+                    .strip_prefix("UID FETCH ")
+                    .and_then(|r| r.split(' ').next())
+                    .and_then(|n| n.parse().ok())
+                    .unwrap_or_else(|| panic!("expected a single-UID fetch, got {cmd}"));
+                fetches.lock().unwrap().push(cmd.clone());
+                write_fetch_message(conn, uid, uid, bodies[(uid - 1) as usize])?;
+                conn.write_line(&format!("{tag} OK"))?;
+            }
+            drain_until_close(conn);
+            Ok(())
+        })
+    };
+    let server = MockImap::start_scripts(vec![control, worker]);
+    let archive = tempfile("byte-chunks");
+    let summary =
+        run_import(&server, "alice", archive, |c| c.fetch_batch_bytes = 30).expect("import");
+    assert_eq!(email_counts(&summary).created, 3, "summary={summary:?}");
+    assert_eq!(
+        fetches.lock().unwrap().len(),
+        3,
+        "one body fetch per message"
+    );
+}
+
+#[test]
+fn a_chunk_larger_than_the_event_queue_completes() {
+    // One worker holds at most two events in flight; a ten-message chunk
+    // has to wait on the archive writer, not deadlock on it.
+    const UIDS: &[u32] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+    let worker: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
+        auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
+        let (tag, _) = conn.read_command()?;
+        write_select(conn, &tag, 999, 11, 10)?;
+        let (tag, cmd) = conn.read_command()?;
+        assert!(cmd.starts_with("UID FETCH 1:10 "), "got {cmd}");
+        for uid in UIDS {
+            let body = format!("From: a@b\r\nMessage-ID: <{uid}@h>\r\n\r\nbody {uid}");
+            write_fetch_message(conn, *uid, *uid, body.as_bytes())?;
+        }
+        conn.write_line(&format!("{tag} OK"))?;
+        drain_until_close(conn);
+        Ok(())
+    });
+    let server = MockImap::start_scripts(vec![control_script_one_folder(999, 11, UIDS), worker]);
+    let archive = tempfile("backpressure");
+    let summary = run_import(&server, "alice", archive, |_| {}).expect("import");
+    assert_eq!(email_counts(&summary).created, 10, "summary={summary:?}");
+}

tests/mock_sync.rs

@@ -487,150 +487,6 @@ fn export_mailbox_already_exists_maps_existing_id() {
     let _ = std::fs::remove_file(&archive);
 }
 
-#[test]
-fn email_export_sends_one_email_per_import_call() {
-    let mut server = mockito::Server::new();
-    let base = server.url();
-    let api = "/jmap/api";
-
-    let archive = tmp();
-    {
-        let conn = db::init::open(&archive).unwrap();
-        conn.execute(
-            "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
-            [],
-        )
-        .unwrap();
-        for n in 1..=2 {
-            let raw =
-                format!("From: a@x\r\nSubject: m{n}\r\nMessage-ID: <m-{n}@h>\r\n\r\nbody {n}",);
-            let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
-            conn.execute(
-                "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
-                 VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')",
-                rusqlite::params![blob],
-            )
-            .unwrap();
-        }
-    }
-
-    let _root = server.mock("GET", "/").with_status(404).create();
-    let _wk = server
-        .mock("GET", "/.well-known/jmap")
-        .with_body(session_body(&base))
-        .create();
-
-    let _mq = server
-        .mock("POST", api)
-        .match_body(Matcher::Regex("Mailbox/query".into()))
-        .with_body(
-            json!({"methodResponses":[["Mailbox/query",
-                {"accountId":"w","ids":["t1"]},"q"]]})
-            .to_string(),
-        )
-        .expect_at_least(1)
-        .create();
-    let _mq_empty = server
-        .mock("POST", api)
-        .match_body(Matcher::AllOf(vec![
-            Matcher::Regex("Mailbox/query".into()),
-            Matcher::Regex("anchor".into()),
-        ]))
-        .with_body(
-            json!({"methodResponses":[["Mailbox/query",
-                {"accountId":"w","ids":[]},"q"]]})
-            .to_string(),
-        )
-        .create();
-    let _mg = server
-        .mock("POST", api)
-        .match_body(Matcher::Regex("Mailbox/get".into()))
-        .with_body(
-            json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
-                {"id":"t1","name":"Inbox","role":"inbox","parentId":null,
-                 "myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
-            .to_string(),
-        )
-        .expect_at_least(1)
-        .create();
-    let _eq = server
-        .mock("POST", api)
-        .match_body(Matcher::Regex("Email/query".into()))
-        .with_body(
-            json!({"methodResponses":[["Email/query",
-                {"accountId":"w","ids":[]},"q"]]})
-            .to_string(),
-        )
-        .expect_at_least(1)
-        .create();
-
-    let _ups = server
-        .mock("POST", Matcher::Regex("/jmap/upload/".into()))
-        .with_body(json!({"blobId":"UP1"}).to_string())
-        .expect(2)
-        .create();
-
-    let single_only = server
-        .mock("POST", api)
-        .match_body(Matcher::AllOf(vec![
-            Matcher::Regex("Email/import".into()),
-            Matcher::Regex("e1".into()),
-            Matcher::Regex("e2".into()),
-        ]))
-        .expect(0)
-        .create();
-
-    let imports = server
-        .mock("POST", api)
-        .match_body(Matcher::Regex("Email/import".into()))
-        .with_body(
-            json!({"methodResponses":[["Email/import",
-                {"accountId":"w","created":{"e":{"id":"x","blobId":"b","threadId":"t","size":10}}},"i"]]})
-            .to_string(),
-        )
-        .expect(2)
-        .create();
-
-    let summary = sync::export::run(
-        CommonConfig {
-            archive: archive.clone(),
-            threads: 1,
-            dry_run: false,
-            max_retries: 0,
-            allow_invalid_certs: false,
-            logger: Logger::from_flags(true, 0),
-        },
-        ExportConfig {
-            connect: ConnectConfig {
-                url: base.clone(),
-                auth: Auth::Basic {
-                    user: "u".into(),
-                    password: "p".into(),
-                },
-                account: AccountSelector::Id("w".into()),
-            },
-            objects: None,
-            prune: false,
-            yes: true,
-        },
-    )
-    .expect("export run");
-
-    let email = summary
-        .per_type
-        .iter()
-        .find(|(t, _)| *t == "Email")
-        .map(|(_, c)| c.clone())
-        .expect("email counts");
-    assert_eq!(email.created, 2, "both emails imported in per-item rounds");
-    assert_eq!(email.failed, 0, "no per-unit failure");
-    assert!(!summary.any_failed(), "no whole-run failure");
-
-    single_only.assert();
-    imports.assert();
-    let _ = std::fs::remove_file(&archive);
-}
-
 #[test]
 fn export_email_blob_not_found_reuploads_and_retries() {
     let mut server = mockito::Server::new();

tests/sync_imap.rs

@@ -71,6 +71,7 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig {
         automap: true,
         include_deleted: false,
         fetch_batch: 256,
+        fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
         imap_connections: 4,
         allow_source_change: false,
     }