Initial commit

src/sync/emailmeta.rs

@@ -0,0 +1,113 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use mail_parser::{Address, HeaderName, HeaderValue, MessageParser};
+
+use crate::sync::keys::{EmailIndex, email_index};
+
+fn addrs(a: Option<&Address>) -> Vec<String> {
+    match a {
+        Some(addr) => addr
+            .iter()
+            .filter_map(|x| x.address().map(|s| s.to_owned()))
+            .collect(),
+        None => Vec::new(),
+    }
+}
+
+fn message_ids(value: &HeaderValue) -> Vec<String> {
+    match value {
+        HeaderValue::Text(s) => vec![s.to_string()],
+        HeaderValue::TextList(l) => l.iter().map(|s| s.to_string()).collect(),
+        _ => Vec::new(),
+    }
+}
+
+pub fn email_index_from_blob(bytes: &[u8]) -> EmailIndex {
+    email_meta_from_blob(bytes).0
+}
+
+pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option<String>) {
+    let Some(m) = MessageParser::default().parse(bytes) else {
+        return (email_index(&[], &[], "", "", &[]), None);
+    };
+    let mut mids: Vec<String> = Vec::new();
+    for v in m.header_values(HeaderName::MessageId) {
+        mids.extend(message_ids(v));
+    }
+    if mids.is_empty()
+        && let Some(single) = m.message_id()
+    {
+        mids.push(single.to_owned());
+    }
+    let date = m.date().map(|d| d.to_rfc3339());
+    let idx = email_index(
+        &mids,
+        &addrs(m.from()),
+        m.subject().unwrap_or(""),
+        date.as_deref().unwrap_or(""),
+        &addrs(m.to()),
+    );
+    (idx, date)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::sync::keys::{EmailKey, email_key};
+
+    #[test]
+    fn parses_core_fields_and_keys_by_message_id() {
+        let raw = b"From: Alice <[email protected]>\r\nTo: Bob <[email protected]>\r\nSubject: Hi there\r\nMessage-ID: <abc-1@host>\r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody\r\n";
+        let idx = email_index_from_blob(raw);
+        assert_eq!(idx.mids, vec!["abc-1@host".to_owned()]);
+        assert_eq!(
+            email_key(&idx),
+            EmailKey::MessageId("abc-1@host".to_owned())
+        );
+    }
+
+    #[test]
+    fn sent_at_offset_is_preserved_then_z_for_utc() {
+        let off = email_index_from_blob(
+            b"Message-ID: <a@h>\r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nx",
+        );
+        let utc = email_index_from_blob(
+            b"Message-ID: <b@h>\r\nDate: Mon, 12 May 2025 08:00:00 +0000\r\n\r\nx",
+        );
+
+        assert_ne!(off.fb, utc.fb);
+    }
+
+    #[test]
+    fn missing_message_id_falls_back_to_hash() {
+        let idx = email_index_from_blob(b"From: a@x\r\nSubject: only\r\n\r\nbody");
+        assert!(idx.mids.is_empty());
+        assert!(matches!(email_key(&idx), EmailKey::Fallback(_)));
+    }
+
+    #[test]
+    fn multiple_message_id_values_are_all_captured() {
+        let idx =
+            email_index_from_blob(b"Message-ID: <a@h>\r\nMessage-ID: <b@h>\r\nSubject: s\r\n\r\nx");
+        assert_eq!(idx.mids, vec!["a@h".to_owned(), "b@h".to_owned()]);
+        assert_eq!(
+            email_key(&idx),
+            EmailKey::MessageId("a@h\u{1f}b@h".to_owned())
+        );
+    }
+
+    #[test]
+    fn same_message_different_received_metadata_same_key() {
+        let a = email_index_from_blob(
+            b"From: a@x\r\nSubject: S\r\nMessage-ID: <dup@h>\r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody",
+        );
+        let b = email_index_from_blob(
+            b"From: a@x\r\nSubject: S\r\nMessage-ID: <DUP@h>\r\nDate: Tue, 13 May 2025 11:00:00 +0000\r\n\r\nbody",
+        );
+        assert_eq!(email_key(&a), email_key(&b));
+    }
+}

src/sync/export.rs

@@ -0,0 +1,501 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+use std::io::{IsTerminal, Write};
+
+use rusqlite::Connection;
+use serde_json::{Map, Value, json};
+
+use crate::db;
+use crate::error::Error;
+use crate::jmap::blobxfer;
+use crate::jmap::connect::{self, Connected};
+use crate::jmap::error::JmapError;
+use crate::jmap::http::HttpClient;
+use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_ids, set_call};
+use crate::jmap::session::{Limits, Session};
+use crate::jmap::wire::JmapId;
+use crate::logging::{LEVEL_DEFAULT, Logger};
+use crate::sync::import_jmap::mapping::{BlobUpload, TargetResolver};
+use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts};
+use crate::types::ObjectType;
+
+const EXPORT_ORDER: [ObjectType; 10] = [
+    ObjectType::Mailbox,
+    ObjectType::AddressBook,
+    ObjectType::Calendar,
+    ObjectType::FileNode,
+    ObjectType::Identity,
+    ObjectType::SieveScript,
+    ObjectType::ParticipantIdentity,
+    ObjectType::Email,
+    ObjectType::ContactCard,
+    ObjectType::CalendarEvent,
+];
+
+type IdMap = HashMap<i64, JmapId>;
+
+#[derive(Default)]
+struct Maps {
+    m: HashMap<ObjectType, IdMap>,
+}
+
+impl Maps {
+    fn insert(&mut self, ty: ObjectType, local: i64, target: JmapId) {
+        self.m.entry(ty).or_default().insert(local, target);
+    }
+}
+
+impl TargetResolver for Maps {
+    fn target(&self, ty: ObjectType, local_id: i64) -> Option<JmapId> {
+        self.m.get(&ty)?.get(&local_id).cloned()
+    }
+}
+
+struct Uploader<'a> {
+    net: &'a Net,
+    conn: &'a Connection,
+    cache: HashMap<i64, JmapId>,
+}
+
+impl<'a> Uploader<'a> {
+    fn new(net: &'a Net, conn: &'a Connection) -> Uploader<'a> {
+        Uploader {
+            net,
+            conn,
+            cache: HashMap::new(),
+        }
+    }
+
+    fn upload_with(&mut self, local_id: i64, content_type: &str) -> Result<JmapId, JmapError> {
+        if let Some(id) = self.cache.get(&local_id) {
+            return Ok(id.clone());
+        }
+        let id = if self.net.dry_run {
+            let _exists = db::blobs::blob_bytes(self.conn, local_id)?
+                .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?;
+            JmapId(format!("dryrun-blob-{local_id}"))
+        } else {
+            let bytes = db::blobs::blob_bytes(self.conn, local_id)?
+                .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?;
+            blobxfer::upload_bytes(
+                &self.net.client,
+                &self.net.session,
+                &self.net.account,
+                content_type,
+                &bytes,
+            )?
+        };
+        self.cache.insert(local_id, id.clone());
+        Ok(id)
+    }
+}
+
+impl BlobUpload for Uploader<'_> {
+    fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> {
+        self.upload_with(local_id, "application/octet-stream")
+    }
+}
+
+#[derive(Clone)]
+struct Net {
+    client: HttpClient,
+    api: String,
+    account: String,
+    limits: Limits,
+    session: Session,
+    dry_run: bool,
+}
+
+fn has_rows(conn: &Connection, ty: ObjectType) -> bool {
+    let table = crate::sync::table_name(ty);
+    conn.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {table})"), [], |r| {
+        r.get::<_, i64>(0)
+    })
+    .map(|n| n != 0)
+    .unwrap_or(false)
+}
+
+pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    let ctx = Context::open(common, &config.connect)?;
+    let connected = connect::prepare(&ctx, &config.connect)?;
+
+    let net = Net {
+        client: ctx.client.clone(),
+        api: connected.session.api_url.clone(),
+        account: connected.account_id.clone(),
+        limits: connected.limits,
+        session: connected.session.clone(),
+        dry_run: ctx.dry_run(),
+    };
+
+    let work = work_list(&ctx.conn, &config, &connected, &logger);
+    let mut maps = Maps::default();
+    let mut summary = Summary::default();
+    let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new();
+    let mut plans: HashMap<ObjectType, Plan> = HashMap::new();
+    let mut counts_per_type: HashMap<ObjectType, TypeCounts> = HashMap::new();
+
+    for ty in &work {
+        if logger.enabled(LEVEL_DEFAULT) {
+            eprintln!("export: {} ...", ty.jmap_name());
+        }
+        let mut counts = TypeCounts::default();
+        let res = reconcile_type(
+            &ctx,
+            &net,
+            *ty,
+            &mut maps,
+            &logger,
+            &mut counts,
+            &mut dry_rows,
+        );
+        let plan = match res {
+            Ok(p) => p,
+            Err(e) => {
+                logger.warn(&format!("type {} aborted: {e}", ty.jmap_name()));
+                counts.failed += 1;
+                Plan::default()
+            }
+        };
+        plans.insert(*ty, plan);
+        counts_per_type.insert(*ty, counts);
+    }
+
+    if config.prune {
+        prune_phase(
+            &ctx,
+            &net,
+            &work,
+            &plans,
+            &config,
+            &logger,
+            &mut counts_per_type,
+        )?;
+    }
+
+    for ty in &work {
+        if let Some(counts) = counts_per_type.remove(ty) {
+            summary.per_type.push((ty.jmap_name(), counts));
+        }
+    }
+
+    if ctx.dry_run() {
+        print_dry_run(&dry_rows, config.prune);
+        return Ok(Summary::default());
+    }
+    summary.retries_observed = ctx.client.retries_observed();
+    summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
+    Ok(summary)
+}
+
+fn prune_phase(
+    ctx: &Context,
+    net: &Net,
+    work: &[ObjectType],
+    plans: &HashMap<ObjectType, Plan>,
+    config: &ExportConfig,
+    logger: &Logger,
+    counts_per_type: &mut HashMap<ObjectType, TypeCounts>,
+) -> Result<(), Error> {
+    let totals: Vec<(ObjectType, &Plan)> = work
+        .iter()
+        .filter_map(|ty| plans.get(ty).map(|p| (*ty, p)))
+        .filter(|(_, p)| !p.prune_candidates.is_empty())
+        .collect();
+    if totals.is_empty() {
+        return Ok(());
+    }
+    eprintln!("prune plan:");
+    let total: usize = totals.iter().map(|(_, p)| p.prune_candidates.len()).sum();
+    for (ty, p) in &totals {
+        eprintln!(
+            "  {:<22} {:>6} candidate(s); sample: {}",
+            ty.jmap_name(),
+            p.prune_candidates.len(),
+            sample(&p.prune_candidates),
+        );
+    }
+    eprintln!("  {:<22} {:>6} total", "(all types)", total);
+    if ctx.dry_run() {
+        return Ok(());
+    }
+    if !config.yes && std::io::stdin().is_terminal() {
+        eprint!("destroy all {total} objects across all types? [y/N] ");
+        let _ = std::io::stderr().flush();
+        let mut line = String::new();
+        std::io::stdin()
+            .read_line(&mut line)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        if !matches!(line.trim(), "y" | "Y" | "yes") {
+            return Err(Error::PruneAborted);
+        }
+    }
+    for ty in work.iter().rev() {
+        if let Some(plan) = plans.get(ty)
+            && !plan.prune_candidates.is_empty()
+            && let Some(counts) = counts_per_type.get_mut(ty)
+        {
+            do_destroy(net, *ty, plan, logger, counts);
+        }
+    }
+    Ok(())
+}
+
+fn work_list(
+    conn: &Connection,
+    config: &ExportConfig,
+    connected: &Connected,
+    logger: &Logger,
+) -> Vec<ObjectType> {
+    let selected = config.objects.as_ref();
+    EXPORT_ORDER
+        .into_iter()
+        .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true))
+        .filter(|ty| has_rows(conn, *ty))
+        .filter(|ty| {
+            if connected.supports(*ty) {
+                true
+            } else {
+                logger.warn(&format!(
+                    "target does not support {}; skipping",
+                    ty.jmap_name()
+                ));
+                false
+            }
+        })
+        .collect()
+}
+
+#[allow(clippy::too_many_arguments)]
+fn reconcile_type(
+    ctx: &Context,
+    net: &Net,
+    ty: ObjectType,
+    maps: &mut Maps,
+    logger: &Logger,
+    counts: &mut TypeCounts,
+    dry_rows: &mut Vec<(&'static str, u64, u64, u64)>,
+) -> Result<Plan, Error> {
+    let plan = match ty {
+        ObjectType::Mailbox | ObjectType::FileNode => {
+            tree::reconcile(ctx, net, ty, maps, counts, logger)
+        }
+        ObjectType::AddressBook | ObjectType::Calendar => {
+            flat::reconcile(ctx, net, ty, maps, counts, logger)
+        }
+        ObjectType::Identity => keyed::reconcile_identity(ctx, net, maps, counts, logger),
+        ObjectType::ParticipantIdentity => {
+            keyed::reconcile_participant(ctx, net, maps, counts, logger)
+        }
+        ObjectType::SieveScript => sieve::reconcile(ctx, net, maps, counts, logger),
+        ObjectType::ContactCard | ObjectType::CalendarEvent => {
+            uidtype::reconcile(ctx, net, ty, maps, counts, logger)
+        }
+        ObjectType::Email => email::reconcile(ctx, net, maps, counts, logger),
+    }?;
+
+    if ctx.dry_run() {
+        dry_rows.push((
+            ty.jmap_name(),
+            counts.created,
+            counts.skipped,
+            plan.prune_candidates.len() as u64,
+        ));
+    }
+
+    Ok(plan)
+}
+
+#[derive(Default)]
+pub struct Plan {
+    pub prune_candidates: Vec<String>,
+    pub active_sieve_target: Option<String>,
+}
+
+fn do_destroy(net: &Net, ty: ObjectType, plan: &Plan, logger: &Logger, counts: &mut TypeCounts) {
+    if ty == ObjectType::SieveScript {
+        deactivate_active_sieve_script(net, logger);
+    }
+    let destroy = Value::Array(
+        plan.prune_candidates
+            .iter()
+            .map(|s| Value::String(s.clone()))
+            .collect(),
+    );
+    let extra = destroy_contents_arg(ty);
+    match set_call(
+        &net.client,
+        &net.api,
+        &net.account,
+        ty.jmap_name(),
+        SetRequest {
+            destroy: Some(destroy),
+            extra_args: &extra,
+            ..Default::default()
+        },
+        &net.limits,
+    ) {
+        Ok(outcome) => {
+            counts.deleted += outcome.destroyed.len() as u64;
+            for (id, err) in &outcome.not_destroyed {
+                logger.warn(&format!(
+                    "prune: {} {id} not destroyed: {err}",
+                    ty.jmap_name()
+                ));
+                counts.skipped += 1;
+            }
+        }
+        Err(e) => {
+            logger.warn(&format!(
+                "prune {}: destroy request failed: {e}",
+                ty.jmap_name()
+            ));
+            counts.failed += plan.prune_candidates.len() as u64;
+        }
+    }
+}
+
+fn deactivate_active_sieve_script(net: &Net, logger: &Logger) {
+    let mut req = Request::new();
+    req.call(
+        "SieveScript/set",
+        json!({ "accountId": net.account, "onSuccessDeactivateScript": true }),
+        "d",
+    );
+    let outcome = req.send(&net.client, &net.api).and_then(|resp| {
+        let mr = resp.first()?;
+        crate::jmap::request::check_method_error(mr)
+    });
+    if let Err(e) = outcome {
+        logger.warn(&format!(
+            "prune: SieveScript deactivation failed before destroy: {e}"
+        ));
+    }
+}
+
+fn destroy_contents_arg(ty: ObjectType) -> Vec<(&'static str, Value)> {
+    match ty {
+        ObjectType::AddressBook => vec![("onDestroyRemoveContents", Value::Bool(false))],
+        ObjectType::Calendar => vec![("onDestroyRemoveEvents", Value::Bool(false))],
+        ObjectType::FileNode => vec![("onDestroyRemoveChildren", Value::Bool(false))],
+        _ => Vec::new(),
+    }
+}
+
+fn sample(ids: &[String]) -> String {
+    let n = ids.len().min(5);
+    ids[..n].join(", ")
+}
+
+fn print_dry_run(rows: &[(&'static str, u64, u64, u64)], prune: bool) {
+    if prune {
+        println!(
+            "{:<22} {:>10} {:>10} {:>12}",
+            "TYPE", "CREATE", "MATCHED", "WOULD-DESTROY"
+        );
+        for (ty, c, m, d) in rows {
+            println!("{ty:<22} {c:>10} {m:>10} {d:>12}");
+        }
+    } else {
+        println!("{:<22} {:>10} {:>10}", "TYPE", "CREATE", "MATCHED");
+        for (ty, c, m, _) in rows {
+            println!("{ty:<22} {c:>10} {m:>10}");
+        }
+    }
+}
+
+mod tree;
+
+mod flat;
+
+mod keyed;
+
+mod sieve;
+
+mod uidtype;
+
+mod email;
+
+mod common {
+    use super::*;
+
+    pub fn target_query_get(
+        net: &Net,
+        ty: ObjectType,
+        props: Option<&[&str]>,
+    ) -> Result<Vec<Value>, JmapError> {
+        let ids = query_all_ids(
+            &net.client,
+            &net.api,
+            &net.account,
+            ty.jmap_name(),
+            &net.limits,
+        )?;
+        if ids.is_empty() {
+            return Ok(Vec::new());
+        }
+        let got = get_objects::<Value>(
+            &net.client,
+            &net.api,
+            &net.account,
+            ty.jmap_name(),
+            &ids,
+            props,
+            &net.limits,
+        )?;
+        Ok(got.list)
+    }
+
+    pub fn target_get_all(net: &Net, ty: ObjectType) -> Result<Vec<Value>, JmapError> {
+        Ok(get_all::<Value>(&net.client, &net.api, &net.account, ty.jmap_name())?.list)
+    }
+
+    pub fn jid(v: &Value) -> Option<String> {
+        v.get("id").and_then(Value::as_str).map(str::to_owned)
+    }
+
+    pub fn create_batch(
+        net: &Net,
+        ty: ObjectType,
+        creates: Vec<(String, Value)>,
+    ) -> Result<crate::jmap::request::SetOutcome, JmapError> {
+        if net.dry_run {
+            return Ok(synthesize_dry_run_outcome(ty, &creates));
+        }
+        let mut map = Map::new();
+        for (cid, obj) in creates {
+            map.insert(cid, obj);
+        }
+        set_call(
+            &net.client,
+            &net.api,
+            &net.account,
+            ty.jmap_name(),
+            SetRequest {
+                create: Some(Value::Object(map)),
+                ..Default::default()
+            },
+            &net.limits,
+        )
+    }
+
+    fn synthesize_dry_run_outcome(
+        ty: ObjectType,
+        creates: &[(String, Value)],
+    ) -> crate::jmap::request::SetOutcome {
+        let mut outcome = crate::jmap::request::SetOutcome::default();
+        for (cid, _) in creates {
+            let synthetic = serde_json::json!({
+                "id": format!("dryrun-{}-{cid}", ty.jmap_name())
+            });
+            outcome.created.push((cid.clone(), synthetic));
+        }
+        outcome
+    }
+}

src/sync/export/email.rs

@@ -0,0 +1,253 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+
+use serde_json::{Map, Value, json};
+
+use super::common::{jid, target_query_get};
+use super::{Maps, Net, Plan, Uploader};
+use crate::error::Error;
+use crate::jmap::error::JmapError;
+use crate::jmap::request::{Request, check_method_error, get_objects};
+use crate::jmap::wire::JmapId;
+use crate::logging::Logger;
+use crate::sync::import_jmap::mapping::{EMAIL_SELECT, TargetResolver, row_to_email};
+use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json};
+use crate::sync::{Context, TypeCounts};
+use crate::types::ObjectType;
+
+fn server_index(v: &Value) -> EmailIndex {
+    let arr = |k: &str| {
+        v.get(k)
+            .and_then(Value::as_array)
+            .map(|a| {
+                a.iter()
+                    .filter_map(|x| x.get("email").and_then(Value::as_str).map(str::to_owned))
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default()
+    };
+    let mids: Vec<String> = v
+        .get("messageId")
+        .and_then(Value::as_array)
+        .map(|a| {
+            a.iter()
+                .filter_map(|x| x.as_str().map(str::to_owned))
+                .collect()
+        })
+        .unwrap_or_default();
+    email_index(
+        &mids,
+        &arr("from"),
+        v.get("subject").and_then(Value::as_str).unwrap_or(""),
+        v.get("sentAt").and_then(Value::as_str).unwrap_or(""),
+        &arr("to"),
+    )
+}
+
+pub fn reconcile(
+    ctx: &Context,
+    net: &Net,
+    maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let ty = ObjectType::Email;
+
+    let target_min = target_query_get(net, ty, Some(&["messageId"])).map_err(Error::from)?;
+    let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
+
+    let fallback_ids: Vec<JmapId> = target_min
+        .iter()
+        .zip(indices.iter())
+        .filter(|(_, i)| i.mids.is_empty())
+        .filter_map(|(v, _)| jid(v).map(JmapId))
+        .collect();
+    if !fallback_ids.is_empty() {
+        let got = get_objects::<Value>(
+            &net.client,
+            &net.api,
+            &net.account,
+            ty.jmap_name(),
+            &fallback_ids,
+            Some(&["messageId", "from", "subject", "sentAt", "to"]),
+            &net.limits,
+        )
+        .map_err(Error::from)?;
+        let by_id: HashMap<String, &Value> = got
+            .list
+            .iter()
+            .filter_map(|v| jid(v).map(|i| (i, v)))
+            .collect();
+        for (v, slot) in target_min.iter().zip(indices.iter_mut()) {
+            if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) {
+                *slot = server_index(full);
+            }
+        }
+    }
+    let target_keys: HashSet<EmailKey> = email_keys(&indices).into_iter().collect();
+
+    let local: Vec<(i64, crate::sync::import_jmap::mapping::EmailRow)> = {
+        let mut stmt = ctx
+            .conn
+            .prepare(EMAIL_SELECT)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        stmt.query_map([], |row| {
+            let id: i64 = row.get(0)?;
+            Ok((id, row_to_email(row)))
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?
+        .into_iter()
+        .map(|(id, r)| Ok((id, r.map_err(Error::from)?)))
+        .collect::<Result<_, Error>>()?
+    };
+
+    let local_indices: Vec<EmailIndex> = local
+        .iter()
+        .map(|(_, r)| index_from_json(&r.message_match))
+        .collect();
+    let local_keys = email_keys(&local_indices);
+
+    let mut uploader = Uploader::new(net, &ctx.conn);
+    for (i, key) in local_keys.iter().enumerate() {
+        if target_keys.contains(key) {
+            counts.skipped += 1;
+            continue;
+        }
+        let (local_id, row) = &local[i];
+        let mut mids = Map::new();
+        let mut all_resolved = true;
+        for ml in &row.mailbox_locals {
+            match maps.target(ObjectType::Mailbox, *ml) {
+                Some(t) => {
+                    mids.insert(t.0, Value::Bool(true));
+                }
+                None => {
+                    all_resolved = false;
+                    break;
+                }
+            }
+        }
+        if !all_resolved {
+            logger.warn(&format!(
+                "email local {local_id} skipped: mailbox not on target"
+            ));
+            counts.failed += 1;
+            continue;
+        }
+        let blob = match uploader.upload_with(row.blob_local_id, "message/rfc822") {
+            Ok(b) => b.0,
+            Err(e) => {
+                logger.warn(&format!("email blob upload failed: {e}"));
+                counts.failed += 1;
+                continue;
+            }
+        };
+        let mut kw = Map::new();
+        for k in &row.keywords {
+            kw.insert(k.clone(), Value::Bool(true));
+        }
+        let item = json!({
+            "blobId": blob,
+            "mailboxIds": Value::Object(mids),
+            "keywords": Value::Object(kw),
+            "receivedAt": row.received_at,
+        });
+        send_import_chunk(net, &[(format!("e{local_id}"), item)], counts, logger);
+    }
+
+    Ok(Plan::default())
+}
+
+fn send_import_chunk(
+    net: &Net,
+    items: &[(String, Value)],
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) {
+    if items.is_empty() {
+        return;
+    }
+    if net.dry_run {
+        counts.created += items.len() as u64;
+        return;
+    }
+    let mut map = Map::new();
+    for (k, v) in items {
+        map.insert(k.clone(), v.clone());
+    }
+    let mut req = Request::new();
+    req.call(
+        "Email/import",
+        json!({ "accountId": net.account, "emails": Value::Object(map) }),
+        "i",
+    );
+    if req.fits(&net.limits).is_err() {
+        resplit_or_fail(net, items, counts, logger);
+        return;
+    }
+    match req.send(&net.client, &net.api) {
+        Ok(resp) => match resp.first().and_then(|mr| {
+            check_method_error(mr)?;
+            Ok(mr)
+        }) {
+            Ok(mr) => absorb_import(mr, counts, logger),
+            Err(JmapError::RequestTooLarge) => resplit_or_fail(net, items, counts, logger),
+            Err(JmapError::Method { error_type, .. }) if error_type == "requestTooLarge" => {
+                resplit_or_fail(net, items, counts, logger);
+            }
+            Err(e) => {
+                logger.warn(&format!(
+                    "Email/import method error ({} items): {e}",
+                    items.len()
+                ));
+                counts.failed += items.len() as u64;
+            }
+        },
+        Err(JmapError::RequestTooLarge) => resplit_or_fail(net, items, counts, logger),
+        Err(e) => {
+            logger.warn(&format!(
+                "Email/import send failed ({} items): {e}",
+                items.len()
+            ));
+            counts.failed += items.len() as u64;
+        }
+    }
+}
+
+fn resplit_or_fail(net: &Net, items: &[(String, Value)], counts: &mut TypeCounts, logger: &Logger) {
+    if items.len() <= 1 {
+        for (cid, _) in items {
+            logger.warn(&format!(
+                "Email/import {cid} exceeds maxSizeRequest alone; skipped"
+            ));
+        }
+        counts.failed += items.len() as u64;
+        return;
+    }
+    let mid = items.len() / 2;
+    send_import_chunk(net, &items[..mid], counts, logger);
+    send_import_chunk(net, &items[mid..], counts, logger);
+}
+
+fn absorb_import(mr: &crate::jmap::request::MethodCall, counts: &mut TypeCounts, logger: &Logger) {
+    if let Some(created) = mr.args.get("created").and_then(Value::as_object) {
+        counts.created += created.len() as u64;
+    }
+    if let Some(nc) = mr.args.get("notCreated").and_then(Value::as_object) {
+        for (cid, err) in nc {
+            let etype = err.get("type").and_then(Value::as_str).unwrap_or("");
+            if etype == "alreadyExists" {
+                counts.skipped += 1;
+            } else {
+                logger.warn(&format!("Email/import {cid} failed: {err}"));
+                counts.failed += 1;
+            }
+        }
+    }
+}

src/sync/export/flat.rs

@@ -0,0 +1,141 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::params;
+use serde_json::{Value, json};
+
+use super::common::{create_batch, jid, target_get_all};
+use super::{Maps, Net, Plan};
+use crate::error::Error;
+use crate::jmap::wire::JmapId;
+use crate::logging::Logger;
+use crate::sync::import_jmap::mapping::{
+    ADDRESS_BOOK_SELECT, CALENDAR_SELECT, row_to_address_book, row_to_calendar,
+};
+use crate::sync::keys::fold_name;
+use crate::sync::prune::{TargetObj, candidates};
+use crate::sync::{Context, TypeCounts};
+use crate::types::ObjectType;
+
+pub fn reconcile(
+    ctx: &Context,
+    net: &Net,
+    ty: ObjectType,
+    maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let table = crate::sync::table_name(ty);
+    let select = if ty == ObjectType::AddressBook {
+        ADDRESS_BOOK_SELECT
+    } else {
+        CALENDAR_SELECT
+    };
+
+    let locals: Vec<(i64, String, bool)> = {
+        let mut stmt = ctx
+            .conn
+            .prepare(&format!("SELECT id, name, is_default FROM {table}"))
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        stmt.query_map([], |r| {
+            Ok((r.get(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)? != 0))
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?
+    };
+
+    let targets = target_get_all(net, ty).map_err(Error::from)?;
+    let mut tmatched = std::collections::HashSet::new();
+
+    let mut to_create: Vec<(i64, bool)> = Vec::new();
+    for (local, name, is_default) in &locals {
+        let hit = targets.iter().find(|t| {
+            let tid = jid(t);
+            t.get("name")
+                .and_then(Value::as_str)
+                .map(|n| fold_name(n) == fold_name(name))
+                .unwrap_or(false)
+                && tid.as_ref().map(|i| !tmatched.contains(i)).unwrap_or(false)
+        });
+        match hit.and_then(jid) {
+            Some(tid) => {
+                tmatched.insert(tid.clone());
+                maps.insert(ty, *local, JmapId(tid));
+                counts.skipped += 1;
+            }
+            None => to_create.push((*local, *is_default)),
+        }
+    }
+
+    if !to_create.is_empty() {
+        let mut batch = Vec::new();
+        for (local, _) in &to_create {
+            let wire = ctx
+                .conn
+                .query_row(&format!("{select} WHERE id = ?1"), params![local], |row| {
+                    Ok(if ty == ObjectType::AddressBook {
+                        row_to_address_book(row).map(|w| serde_json::to_value(&w))
+                    } else {
+                        row_to_calendar(row).map(|w| serde_json::to_value(&w))
+                    })
+                })
+                .map_err(|e| Error::Partial(e.to_string()))?
+                .map_err(Error::from)?
+                .map_err(|e| Error::Partial(e.to_string()))?;
+            let mut wire = wire;
+            if let Value::Object(m) = &mut wire {
+                m.remove("isDefault");
+            }
+            batch.push((format!("c{local}"), wire));
+        }
+        let outcome = create_batch(net, ty, batch).map_err(Error::from)?;
+        for (cid, v) in &outcome.created {
+            if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
+                && let Some(id) = jid(v)
+            {
+                maps.insert(ty, local, JmapId(id.clone()));
+                counts.created += 1;
+                if to_create.iter().any(|(l, d)| *l == local && *d) {
+                    let mut req = crate::jmap::request::Request::new();
+                    req.call(
+                        format!("{}/set", ty.jmap_name()),
+                        json!({ "accountId": net.account, "onSuccessSetIsDefault": id }),
+                        "d",
+                    );
+                    if let Err(e) = req.send(&net.client, &net.api) {
+                        logger.warn(&format!("{} isDefault not set: {e}", ty.jmap_name()));
+                    }
+                }
+            }
+        }
+        for (cid, err) in &outcome.not_created {
+            logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name()));
+            counts.failed += 1;
+        }
+    }
+
+    let objs: Vec<TargetObj> = targets
+        .iter()
+        .filter_map(|t| {
+            let id = jid(t)?;
+            Some(TargetObj {
+                id: id.clone(),
+                matched: tmatched.contains(&id),
+                protected: t.get("isDefault").and_then(Value::as_bool).unwrap_or(false),
+                may_delete: t
+                    .get("myRights")
+                    .and_then(|r| r.get("mayDelete"))
+                    .and_then(Value::as_bool)
+                    .unwrap_or(true),
+                parent: None,
+            })
+        })
+        .collect();
+    Ok(Plan {
+        prune_candidates: candidates(&objs, false),
+        active_sieve_target: None,
+    })
+}

src/sync/export/keyed.rs

@@ -0,0 +1,145 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashSet;
+
+use serde_json::Value;
+
+use super::common::{create_batch, target_get_all};
+use super::{Maps, Net, Plan};
+use crate::error::Error;
+use crate::logging::Logger;
+use crate::sync::import_jmap::mapping::{
+    IDENTITY_SELECT, PARTICIPANT_IDENTITY_SELECT, row_to_identity, row_to_participant_identity,
+};
+use crate::sync::keys::{identity_key, participant_identity_key};
+use crate::sync::{Context, TypeCounts};
+use crate::types::ObjectType;
+
+fn run(
+    net: &Net,
+    ty: ObjectType,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+    target_key: impl Fn(&Value) -> Option<[u8; 32]>,
+    local_rows: Vec<(i64, [u8; 32], Value)>,
+) -> Result<Plan, Error> {
+    let targets = target_get_all(net, ty).map_err(Error::from)?;
+    let mut target_keys: HashSet<[u8; 32]> = HashSet::new();
+    for t in &targets {
+        if let Some(k) = target_key(t) {
+            target_keys.insert(k);
+        }
+    }
+
+    let mut seen: HashSet<[u8; 32]> = HashSet::new();
+    let mut batch = Vec::new();
+    for (local, key, wire) in local_rows {
+        if target_keys.contains(&key) || !seen.insert(key) {
+            counts.skipped += 1;
+            continue;
+        }
+        batch.push((format!("c{local}"), wire));
+    }
+    if !batch.is_empty() {
+        let outcome = create_batch(net, ty, batch).map_err(Error::from)?;
+        counts.created += outcome.created.len() as u64;
+        for (cid, err) in &outcome.not_created {
+            logger.warn(&format!(
+                "{} {cid} not created (expected for non-owned addresses): {err}",
+                ty.jmap_name()
+            ));
+            counts.skipped += 1;
+        }
+    }
+    Ok(Plan::default())
+}
+
+pub fn reconcile_identity(
+    ctx: &Context,
+    net: &Net,
+    _maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let mut stmt = ctx
+        .conn
+        .prepare(IDENTITY_SELECT)
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let rows: Vec<(i64, [u8; 32], Value)> = stmt
+        .query_map([], |row| {
+            let id: i64 = row.get(0)?;
+            Ok((id, row_to_identity(row)))
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?
+        .into_iter()
+        .map(|(id, w)| {
+            let w = w.map_err(Error::from)?;
+            let key = identity_key(&w.name, &w.email);
+            let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?;
+            Ok((id, key, v))
+        })
+        .collect::<Result<_, Error>>()?;
+    drop(stmt);
+    run(
+        net,
+        ObjectType::Identity,
+        counts,
+        logger,
+        |t| {
+            Some(identity_key(
+                t.get("name").and_then(Value::as_str).unwrap_or(""),
+                t.get("email").and_then(Value::as_str).unwrap_or(""),
+            ))
+        },
+        rows,
+    )
+}
+
+pub fn reconcile_participant(
+    ctx: &Context,
+    net: &Net,
+    _maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let mut stmt = ctx
+        .conn
+        .prepare(PARTICIPANT_IDENTITY_SELECT)
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let rows: Vec<(i64, [u8; 32], Value)> = stmt
+        .query_map([], |row| {
+            let id: i64 = row.get(0)?;
+            Ok((id, row_to_participant_identity(row)))
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?
+        .into_iter()
+        .map(|(id, w)| {
+            let w = w.map_err(Error::from)?;
+            let key = participant_identity_key(&w.calendar_address, &w.name);
+            let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?;
+            Ok((id, key, v))
+        })
+        .collect::<Result<_, Error>>()?;
+    drop(stmt);
+    run(
+        net,
+        ObjectType::ParticipantIdentity,
+        counts,
+        logger,
+        |t| {
+            Some(participant_identity_key(
+                t.get("calendarAddress")
+                    .and_then(Value::as_str)
+                    .unwrap_or(""),
+                t.get("name").and_then(Value::as_str).unwrap_or(""),
+            ))
+        },
+        rows,
+    )
+}

src/sync/export/sieve.rs

@@ -0,0 +1,151 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use serde_json::{Value, json};
+
+use super::common::{create_batch, jid, target_get_all};
+use super::{Maps, Net, Plan, Uploader};
+use crate::db;
+use crate::error::Error;
+use crate::jmap::blobxfer;
+use crate::jmap::request::Request;
+use crate::logging::Logger;
+use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script};
+use crate::sync::keys::blake3_bytes;
+use crate::sync::{Context, TypeCounts};
+use crate::types::ObjectType;
+
+pub fn reconcile(
+    ctx: &Context,
+    net: &Net,
+    _maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let ty = ObjectType::SieveScript;
+    let targets = target_get_all(net, ty).map_err(Error::from)?;
+
+    let mut target_by_key: HashMap<[u8; 32], String> = HashMap::new();
+    for t in &targets {
+        let (Some(id), Some(blob)) = (jid(t), t.get("blobId").and_then(Value::as_str)) else {
+            continue;
+        };
+        let bytes = blobxfer::download_bytes(
+            &net.client,
+            &net.session,
+            &net.account,
+            blob,
+            "application/sieve",
+            "script",
+        )
+        .map_err(Error::from)?;
+        target_by_key.insert(blake3_bytes(&bytes), id);
+    }
+
+    let locals: Vec<(i64, Option<String>, bool, i64)> = {
+        let mut stmt = ctx
+            .conn
+            .prepare(SIEVE_SELECT)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        stmt.query_map([], |row| {
+            let sr = row_to_sieve_script(row);
+            Ok((row.get::<_, i64>(0)?, sr))
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?
+        .into_iter()
+        .map(|(id, sr)| {
+            let sr = sr.map_err(Error::from)?;
+            Ok((id, sr.name, sr.is_active, sr.blob_local_id))
+        })
+        .collect::<Result<_, Error>>()?
+    };
+
+    let mut active_target: Option<String> = None;
+    let mut deactivate = false;
+    let mut uploader = Uploader::new(net, &ctx.conn);
+
+    for (local, name, is_active, blob_local) in &locals {
+        let bytes = db::blobs::blob_bytes(&ctx.conn, *blob_local)
+            .map_err(|e| Error::Partial(e.to_string()))?
+            .ok_or_else(|| Error::Partial("sieve blob missing".to_owned()))?;
+        let key = blake3_bytes(&bytes);
+        let target_id = if let Some(id) = target_by_key.get(&key) {
+            counts.skipped += 1;
+            id.clone()
+        } else {
+            let blob_id = uploader
+                .upload_with(*blob_local, "application/sieve")
+                .map_err(Error::from)?;
+            let mut obj = serde_json::Map::new();
+            if let Some(n) = name {
+                obj.insert("name".to_owned(), Value::String(n.clone()));
+            }
+            obj.insert("blobId".to_owned(), Value::String(blob_id.0));
+            let outcome = create_batch(net, ty, vec![(format!("c{local}"), Value::Object(obj))])
+                .map_err(Error::from)?;
+            match outcome.created.first().and_then(|(_, v)| jid(v)) {
+                Some(id) => {
+                    counts.created += 1;
+                    target_by_key.insert(key, id.clone());
+                    id
+                }
+                None => {
+                    for (cid, err) in &outcome.not_created {
+                        logger.warn(&format!("SieveScript {cid} not created: {err}"));
+                    }
+                    counts.failed += 1;
+                    continue;
+                }
+            }
+        };
+        if *is_active {
+            active_target = Some(target_id);
+        }
+    }
+
+    if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) {
+        deactivate = true;
+    }
+
+    if !net.dry_run {
+        let mut req = Request::new();
+        let args = if let Some(id) = &active_target {
+            json!({ "accountId": net.account, "onSuccessActivateScript": id })
+        } else if deactivate {
+            json!({ "accountId": net.account, "onSuccessDeactivateScript": true })
+        } else {
+            json!({ "accountId": net.account })
+        };
+        req.call("SieveScript/set", args, "a");
+        if let Err(e) = req.send(&net.client, &net.api) {
+            logger.warn(&format!("SieveScript activation failed: {e}"));
+        }
+    }
+
+    let local_keys: std::collections::HashSet<[u8; 32]> = locals
+        .iter()
+        .filter_map(|(_, _, _, b)| {
+            db::blobs::blob_bytes(&ctx.conn, *b)
+                .ok()
+                .flatten()
+                .map(|by| blake3_bytes(&by))
+        })
+        .collect();
+    let mut prune_candidates: Vec<String> = target_by_key
+        .iter()
+        .filter(|(k, _)| !local_keys.contains(*k))
+        .map(|(_, id)| id.clone())
+        .collect();
+    prune_candidates.sort();
+
+    Ok(Plan {
+        prune_candidates,
+        active_sieve_target: active_target,
+    })
+}

src/sync/export/tree.rs

@@ -0,0 +1,321 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+
+use rusqlite::params;
+use serde_json::Value;
+
+use super::common::{create_batch, jid, target_query_get};
+use super::{Net, Plan, Uploader};
+use crate::error::Error;
+use crate::jmap::wire::JmapId;
+use crate::logging::Logger;
+use crate::sync::import_jmap::mapping::{
+    FILE_NODE_SELECT, MAILBOX_SELECT, TargetResolver, row_to_file_node, row_to_mailbox,
+};
+use crate::sync::keys::fold_name;
+use crate::sync::{Context, TypeCounts};
+use crate::types::ObjectType;
+
+use super::Maps;
+
+struct LocalNode {
+    local: i64,
+    parent: Option<i64>,
+    name: String,
+    role: Option<String>,
+}
+
+struct TargetNode {
+    id: String,
+    parent: Option<String>,
+    name: String,
+    role: Option<String>,
+    may_delete: bool,
+}
+
+fn load_local(ctx: &Context, ty: ObjectType) -> Result<Vec<LocalNode>, Error> {
+    let table = crate::sync::table_name(ty);
+    let mut stmt = ctx
+        .conn
+        .prepare(&format!("SELECT id, parent_id, name, role FROM {table}"))
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let rows = stmt
+        .query_map([], |r| {
+            Ok(LocalNode {
+                local: r.get(0)?,
+                parent: r.get(1)?,
+                name: r.get(2)?,
+                role: r.get(3)?,
+            })
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(rows)
+}
+
+fn load_target(net: &Net, ty: ObjectType) -> Result<Vec<TargetNode>, Error> {
+    let props: &[&str] = &["role", "name", "parentId", "myRights"];
+    let list = target_query_get(net, ty, Some(props)).map_err(Error::from)?;
+    Ok(list
+        .iter()
+        .filter_map(|v| {
+            Some(TargetNode {
+                id: jid(v)?,
+                parent: v.get("parentId").and_then(Value::as_str).map(str::to_owned),
+                name: v.get("name").and_then(Value::as_str)?.to_owned(),
+                role: v.get("role").and_then(Value::as_str).map(str::to_owned),
+                may_delete: v
+                    .get("myRights")
+                    .and_then(|r| r.get("mayDelete"))
+                    .and_then(Value::as_bool)
+                    .unwrap_or(true),
+            })
+        })
+        .collect())
+}
+
+fn depth(local: i64, by: &HashMap<i64, Option<i64>>) -> usize {
+    let mut d = 0;
+    let mut cur = by.get(&local).copied().flatten();
+    let mut seen = HashSet::new();
+    while let Some(p) = cur {
+        if !seen.insert(p) {
+            break;
+        }
+        d += 1;
+        cur = by.get(&p).copied().flatten();
+    }
+    d
+}
+
+pub fn reconcile(
+    ctx: &Context,
+    net: &Net,
+    ty: ObjectType,
+    maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let locals = load_local(ctx, ty)?;
+    let targets = load_target(net, ty)?;
+
+    let mut matched: HashMap<i64, String> = HashMap::new();
+    let mut tmatched: HashSet<String> = HashSet::new();
+
+    if ty == ObjectType::Mailbox {
+        for t in &targets {
+            if let Some(r) = &t.role
+                && let Some(l) = locals.iter().find(|l| {
+                    l.role.as_deref() == Some(r.as_str()) && !matched.contains_key(&l.local)
+                })
+            {
+                matched.insert(l.local, t.id.clone());
+                tmatched.insert(t.id.clone());
+            }
+        }
+    }
+
+    let mut stack: Vec<(Option<i64>, Option<String>)> = vec![(None, None)];
+    for (l, tid) in matched.clone() {
+        stack.push((Some(l), Some(tid)));
+    }
+    while let Some((lp, tp)) = stack.pop() {
+        let lchildren: Vec<&LocalNode> = locals
+            .iter()
+            .filter(|n| n.parent == lp && !matched.contains_key(&n.local))
+            .collect();
+        for ln in lchildren {
+            if let Some(tn) = targets.iter().find(|t| {
+                t.parent.as_deref() == tp.as_deref()
+                    && !tmatched.contains(&t.id)
+                    && fold_name(&t.name) == fold_name(&ln.name)
+            }) {
+                matched.insert(ln.local, tn.id.clone());
+                tmatched.insert(tn.id.clone());
+                stack.push((Some(ln.local), Some(tn.id.clone())));
+            }
+        }
+    }
+
+    for (l, tid) in &matched {
+        maps.insert(ty, *l, JmapId(tid.clone()));
+    }
+    counts.skipped += matched.len() as u64;
+
+    let by: HashMap<i64, Option<i64>> = locals.iter().map(|n| (n.local, n.parent)).collect();
+    let mut to_create: Vec<&LocalNode> = locals
+        .iter()
+        .filter(|n| !matched.contains_key(&n.local))
+        .collect();
+    to_create.sort_by_key(|n| depth(n.local, &by));
+    let max_depth = to_create
+        .iter()
+        .map(|n| depth(n.local, &by))
+        .max()
+        .unwrap_or(0);
+
+    let mut uploader = Uploader::new(net, &ctx.conn);
+    let interleave = ty == ObjectType::FileNode;
+    for d in 0..=max_depth {
+        let level: Vec<&LocalNode> = to_create
+            .iter()
+            .copied()
+            .filter(|n| depth(n.local, &by) == d)
+            .collect();
+        if interleave {
+            for n in &level {
+                if let Some(p) = n.parent
+                    && maps.target(ty, p).is_none()
+                {
+                    logger.warn(&format!(
+                        "{} local {} skipped: parent {} not created",
+                        ty.jmap_name(),
+                        n.local,
+                        p
+                    ));
+                    counts.failed += 1;
+                    continue;
+                }
+                let obj = match build_create(ctx, ty, n.local, maps, &mut uploader) {
+                    Ok(o) => o,
+                    Err(e) => {
+                        logger.warn(&format!(
+                            "{} local {} skipped: {e}",
+                            ty.jmap_name(),
+                            n.local
+                        ));
+                        counts.failed += 1;
+                        continue;
+                    }
+                };
+                let outcome = create_batch(net, ty, vec![(format!("c{}", n.local), obj)])
+                    .map_err(Error::from)?;
+                for (cid, v) in &outcome.created {
+                    if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
+                        && let Some(id) = jid(v)
+                    {
+                        maps.insert(ty, local, JmapId(id));
+                        counts.created += 1;
+                    }
+                }
+                for (cid, err) in &outcome.not_created {
+                    logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name()));
+                    counts.failed += 1;
+                }
+            }
+            continue;
+        }
+        let mut batch: Vec<(String, Value)> = Vec::new();
+        for n in &level {
+            if let Some(p) = n.parent
+                && maps.target(ty, p).is_none()
+            {
+                logger.warn(&format!(
+                    "{} local {} skipped: parent {} not created",
+                    ty.jmap_name(),
+                    n.local,
+                    p
+                ));
+                counts.failed += 1;
+                continue;
+            }
+            match build_create(ctx, ty, n.local, maps, &mut uploader) {
+                Ok(obj) => {
+                    batch.push((format!("c{}", n.local), obj));
+                }
+                Err(e) => {
+                    logger.warn(&format!(
+                        "{} local {} skipped: {e}",
+                        ty.jmap_name(),
+                        n.local
+                    ));
+                    counts.failed += 1;
+                }
+            }
+        }
+        if batch.is_empty() {
+            continue;
+        }
+        let outcome = create_batch(net, ty, batch).map_err(Error::from)?;
+        for (cid, v) in &outcome.created {
+            if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
+                && let Some(id) = jid(v)
+            {
+                maps.insert(ty, local, JmapId(id));
+                counts.created += 1;
+            }
+        }
+        for (cid, err) in &outcome.not_created {
+            logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name()));
+            counts.failed += 1;
+        }
+    }
+
+    let objs: Vec<crate::sync::prune::TargetObj> = targets
+        .iter()
+        .map(|t| crate::sync::prune::TargetObj {
+            id: t.id.clone(),
+            matched: tmatched.contains(&t.id),
+            protected: t.role.is_some(),
+            may_delete: t.may_delete,
+            parent: t.parent.clone(),
+        })
+        .collect();
+    Ok(Plan {
+        prune_candidates: crate::sync::prune::candidates(&objs, true),
+        active_sieve_target: None,
+    })
+}
+
+fn build_create(
+    ctx: &Context,
+    ty: ObjectType,
+    local: i64,
+    maps: &Maps,
+    uploader: &mut Uploader<'_>,
+) -> Result<Value, Error> {
+    if ty == ObjectType::Mailbox {
+        let wire = ctx
+            .conn
+            .query_row(
+                &format!("{MAILBOX_SELECT} WHERE id = ?1"),
+                params![local],
+                |row| Ok(row_to_mailbox(row, maps)),
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?
+            .map_err(Error::from)?;
+        return serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()));
+    }
+    let fnrow = ctx
+        .conn
+        .query_row(
+            &format!("{FILE_NODE_SELECT} WHERE id = ?1"),
+            params![local],
+            |row| Ok(row_to_file_node(row, maps)),
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        .map_err(Error::from)?;
+    let mut wire = fnrow.wire;
+    if let (crate::jmap::wire::file_node::NodeType::File, Some(blob_local)) =
+        (&wire.node_type, fnrow.blob_local_id)
+    {
+        let ct = wire
+            .media_type
+            .clone()
+            .unwrap_or_else(|| "application/octet-stream".to_owned());
+        let id = uploader.upload_with(blob_local, &ct).map_err(Error::from)?;
+        wire.blob_id = Some(id);
+    }
+    let mut obj = serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()))?;
+    if let Value::Object(m) = &mut obj {
+        m.remove("created");
+        m.remove("modified");
+        m.remove("isSubscribed");
+    }
+    Ok(obj)
+}

src/sync/export/uidtype.rs

@@ -0,0 +1,149 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashSet;
+
+use serde_json::Value;
+
+use super::common::{create_batch, jid, target_query_get};
+use super::{Maps, Net, Plan, Uploader};
+use crate::error::Error;
+use crate::logging::Logger;
+use crate::sync::import_jmap::mapping::{
+    CALENDAR_EVENT_SELECT, CONTACT_CARD_SELECT, calendar_event_to_wire, contact_card_to_wire,
+};
+use crate::sync::prune::{TargetObj, candidates};
+use crate::sync::{Context, TypeCounts};
+use crate::types::ObjectType;
+
+fn target_uid(v: &Value) -> Option<String> {
+    v.get("uid").and_then(Value::as_str).map(str::to_owned)
+}
+
+pub fn reconcile(
+    ctx: &Context,
+    net: &Net,
+    ty: ObjectType,
+    maps: &mut Maps,
+    counts: &mut TypeCounts,
+    logger: &Logger,
+) -> Result<Plan, Error> {
+    let targets = target_query_get(net, ty, None).map_err(Error::from)?;
+    let mut by_uid: std::collections::HashMap<String, String> = std::collections::HashMap::new();
+    for t in &targets {
+        if let (Some(uid), Some(id)) = (target_uid(t), jid(t)) {
+            by_uid.entry(uid).or_insert(id);
+        }
+    }
+
+    let select = if ty == ObjectType::ContactCard {
+        CONTACT_CARD_SELECT
+    } else {
+        CALENDAR_EVENT_SELECT
+    };
+    let rows: Vec<(i64, String)> = {
+        let mut stmt = ctx
+            .conn
+            .prepare(select)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        stmt.query_map([], |r| {
+            if ty == ObjectType::ContactCard {
+                Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))
+            } else {
+                let data: String = r.get(4)?;
+                let uid = serde_json::from_str::<Value>(&data)
+                    .ok()
+                    .and_then(|v| v.get("uid").and_then(Value::as_str).map(str::to_owned))
+                    .unwrap_or_default();
+                Ok((r.get::<_, i64>(0)?, uid))
+            }
+        })
+        .and_then(|m| m.collect::<Result<Vec<_>, _>>())
+        .map_err(|e| Error::Partial(e.to_string()))?
+    };
+
+    let mut matched_uids: HashSet<String> = HashSet::new();
+    let mut uploader = Uploader::new(net, &ctx.conn);
+    for (local, uid) in &rows {
+        if let Some(tid) = by_uid.get(uid) {
+            maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
+            matched_uids.insert(uid.clone());
+            counts.skipped += 1;
+            continue;
+        }
+        let wire = match build_wire(ctx, ty, *local, maps, &mut uploader) {
+            Ok(w) => w,
+            Err(e) => {
+                logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name()));
+                counts.failed += 1;
+                continue;
+            }
+        };
+        let outcome =
+            create_batch(net, ty, vec![(format!("c{local}"), wire)]).map_err(Error::from)?;
+        for (cid, v) in &outcome.created {
+            if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
+                && let Some(id) = jid(v)
+            {
+                maps.insert(ty, parsed, crate::jmap::wire::JmapId(id));
+                counts.created += 1;
+            }
+        }
+        for (cid, err) in &outcome.not_created {
+            logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name()));
+            counts.failed += 1;
+        }
+    }
+
+    let objs: Vec<TargetObj> = targets
+        .iter()
+        .filter_map(|t| {
+            let id = jid(t)?;
+            let uid = target_uid(t);
+            Some(TargetObj {
+                id: id.clone(),
+                matched: uid.map(|u| matched_uids.contains(&u)).unwrap_or(false),
+                protected: false,
+                may_delete: true,
+                parent: None,
+            })
+        })
+        .collect();
+    Ok(Plan {
+        prune_candidates: candidates(&objs, false),
+        active_sieve_target: None,
+    })
+}
+
+fn build_wire(
+    ctx: &Context,
+    ty: ObjectType,
+    local: i64,
+    maps: &Maps,
+    up: &mut Uploader<'_>,
+) -> Result<Value, Error> {
+    if ty == ObjectType::ContactCard {
+        let (uid, abids, data): (String, String, String) = ctx
+            .conn
+            .query_row(
+                &format!("{CONTACT_CARD_SELECT} WHERE id = ?1"),
+                rusqlite::params![local],
+                |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)),
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        contact_card_to_wire(&uid, &abids, &data, maps, up).map_err(Error::from)
+    } else {
+        let (cal, dr, ud, data): (String, i64, i64, String) = ctx
+            .conn
+            .query_row(
+                &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"),
+                rusqlite::params![local],
+                |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, up).map_err(Error::from)
+    }
+}

src/sync/import_dav.rs

@@ -0,0 +1,13 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod calcard;
+pub mod collections;
+pub mod coordinator;
+pub mod items;
+pub mod tree;
+
+pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run};

src/sync/import_dav/calcard.rs

@@ -0,0 +1,281 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use calcard::icalendar::ICalendar;
+use calcard::vcard::VCard;
+use serde_json::{Map, Value};
+
+#[derive(Debug, thiserror::Error)]
+pub enum CalcardError {
+    #[error("vcard parse failed")]
+    VCardParse,
+    #[error("icalendar parse failed")]
+    ICalParse,
+    #[error("jscontact serialisation: {0}")]
+    Json(#[from] serde_json::Error),
+    #[error("no parseable entries in iCalendar")]
+    NoEntries,
+    #[error("calcard output is not a JSON object")]
+    NotAnObject,
+}
+
+#[derive(Debug)]
+pub struct JsContact {
+    pub uid: String,
+    pub data: Value,
+}
+
+pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result<JsContact, CalcardError> {
+    let vcard = VCard::parse(text).map_err(|_| CalcardError::VCardParse)?;
+    let js = vcard.into_jscontact::<String, String>();
+    let value: Value = serde_json::from_str(&js.to_string_pretty())?;
+    let uid = value
+        .get("uid")
+        .and_then(Value::as_str)
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+        .map(str::to_owned)
+        .unwrap_or_else(|| synthesise_uid(item_href));
+    let mut obj = match value {
+        Value::Object(map) => map,
+        _ => return Err(CalcardError::NotAnObject),
+    };
+    obj.remove("uid");
+    Ok(JsContact {
+        uid,
+        data: Value::Object(obj),
+    })
+}
+
+pub fn synthesise_uid(item_href: &str) -> String {
+    let hash = blake3::hash(item_href.as_bytes());
+    format!("vandelay-syn-{}", hash.to_hex())
+}
+
+#[derive(Debug)]
+pub struct JsCalendarEntry {
+    pub data: Value,
+    pub data_type: EntryType,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EntryType {
+    Event,
+    Task,
+    Note,
+    Group,
+}
+
+impl EntryType {
+    pub fn as_column(self) -> &'static str {
+        match self {
+            EntryType::Event => "Event",
+            EntryType::Task => "Task",
+            EntryType::Note => "Note",
+            EntryType::Group => "Group",
+        }
+    }
+}
+
+pub fn ical_to_jscalendar_entries(text: &str) -> Result<Vec<JsCalendarEntry>, CalcardError> {
+    let ical = ICalendar::parse(text).map_err(|_| CalcardError::ICalParse)?;
+    let js = ical.into_jscalendar::<String, String>();
+    let value: Value = serde_json::from_str(&js.to_string_pretty())?;
+    let mut out = Vec::new();
+    collect_entries(value, &mut out);
+    if out.is_empty() {
+        return Err(CalcardError::NoEntries);
+    }
+    Ok(out)
+}
+
+fn collect_entries(value: Value, out: &mut Vec<JsCalendarEntry>) {
+    match value {
+        Value::Object(map) => {
+            let at_type = map.get("@type").and_then(Value::as_str).map(str::to_owned);
+            match at_type.as_deref() {
+                Some("Event") if has_nonempty_uid(&map) => out.push(JsCalendarEntry {
+                    data: Value::Object(map),
+                    data_type: EntryType::Event,
+                }),
+                Some("Task") if has_nonempty_uid(&map) => out.push(JsCalendarEntry {
+                    data: Value::Object(map),
+                    data_type: EntryType::Task,
+                }),
+                Some("Note") if has_nonempty_uid(&map) => out.push(JsCalendarEntry {
+                    data: Value::Object(map),
+                    data_type: EntryType::Note,
+                }),
+                Some("Group") => {
+                    if let Some(entries) = map.get("entries").and_then(Value::as_array) {
+                        for entry in entries {
+                            collect_entries(entry.clone(), out);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        Value::Array(items) => {
+            for item in items {
+                collect_entries(item, out);
+            }
+        }
+        _ => {}
+    }
+}
+
+fn has_nonempty_uid(map: &Map<String, Value>) -> bool {
+    map.get("uid")
+        .and_then(Value::as_str)
+        .map(|u| !u.trim().is_empty())
+        .unwrap_or(false)
+}
+
+pub fn strip_extracted_fields_from_event(value: &mut Value) -> (bool, bool, Option<String>) {
+    let Value::Object(map) = value else {
+        return (false, false, None);
+    };
+    let is_draft = map
+        .remove("isDraft")
+        .and_then(|v| v.as_bool())
+        .unwrap_or(false);
+    let use_default_alerts = map
+        .remove("useDefaultAlerts")
+        .and_then(|v| v.as_bool())
+        .unwrap_or(false);
+    map.remove("calendarIds");
+    map.remove("method");
+    map.remove("utcStart");
+    map.remove("utcEnd");
+    map.remove("isOrigin");
+    map.remove("baseEventId");
+    let uid = map.get("uid").and_then(Value::as_str).map(str::to_owned);
+    (is_draft, use_default_alerts, uid)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const SAMPLE_VCARD: &str = "BEGIN:VCARD\r\n\
+VERSION:4.0\r\n\
+UID:abc-123\r\n\
+FN:Alice Smith\r\n\
+EMAIL:[email protected]\r\n\
+END:VCARD\r\n";
+
+    const SAMPLE_ICAL: &str = "BEGIN:VCALENDAR\r\n\
+VERSION:2.0\r\n\
+PRODID:-//Test//EN\r\n\
+BEGIN:VEVENT\r\n\
+UID:[email protected]\r\n\
+DTSTAMP:20260101T000000Z\r\n\
+DTSTART:20260101T090000Z\r\n\
+DTEND:20260101T100000Z\r\n\
+SUMMARY:Test Event\r\n\
+END:VEVENT\r\n\
+END:VCALENDAR\r\n";
+
+    #[test]
+    fn vcard_extracts_uid_into_separate_field() {
+        let jc = vcard_to_jscontact(SAMPLE_VCARD, "/dav/card/u/d/a.vcf").expect("parse");
+        assert_eq!(jc.uid, "abc-123");
+        let obj = jc.data.as_object().unwrap();
+        assert!(!obj.contains_key("uid"), "uid stripped from data");
+    }
+
+    #[test]
+    fn ical_extracts_at_least_one_event() {
+        let events = ical_to_jscalendar_entries(SAMPLE_ICAL).expect("parse");
+        assert!(!events.is_empty());
+        assert_eq!(events[0].data_type, EntryType::Event);
+        let uid = events[0]
+            .data
+            .as_object()
+            .and_then(|m| m.get("uid"))
+            .and_then(Value::as_str)
+            .unwrap();
+        assert!(uid.starts_with("event-1"));
+    }
+
+    #[test]
+    fn vcard_without_uid_gets_synthetic_uid_from_href() {
+        let no_uid = "BEGIN:VCARD\r\nVERSION:4.0\r\nFN:Anon\r\nEND:VCARD\r\n";
+        let href = "/dav/card/u/d/no-uid.vcf";
+        let jc = vcard_to_jscontact(no_uid, href).expect("synthetic uid");
+        assert_eq!(jc.uid, synthesise_uid(href));
+        assert!(jc.uid.starts_with("vandelay-syn-"));
+    }
+
+    #[test]
+    fn synthetic_uid_is_stable_per_href() {
+        assert_eq!(synthesise_uid("/a"), synthesise_uid("/a"));
+        assert_ne!(synthesise_uid("/a"), synthesise_uid("/b"));
+    }
+
+    #[test]
+    fn ical_with_empty_calendar_fails_no_entries() {
+        let empty = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n";
+        let err = ical_to_jscalendar_entries(empty).unwrap_err();
+        assert!(matches!(err, CalcardError::NoEntries));
+    }
+
+    #[test]
+    fn vcard_garbage_fails_parse() {
+        let err = vcard_to_jscontact("not a vcard", "/x").unwrap_err();
+        assert!(matches!(err, CalcardError::VCardParse));
+    }
+
+    #[test]
+    fn ical_vtodo_classified_as_task() {
+        let vtodo = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\
+            BEGIN:VTODO\r\nUID:t1@example\r\nDTSTAMP:20260101T000000Z\r\n\
+            SUMMARY:Buy milk\r\nEND:VTODO\r\nEND:VCALENDAR\r\n";
+        let entries = ical_to_jscalendar_entries(vtodo).expect("parse vtodo");
+        assert_eq!(entries[0].data_type, EntryType::Task);
+    }
+
+    #[test]
+    fn ical_recurrence_overrides_collapse_to_single_event() {
+        let ical = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\
+            BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\
+            DTSTART:20260101T090000Z\r\nDTEND:20260101T100000Z\r\n\
+            RRULE:FREQ=DAILY;COUNT=3\r\nSUMMARY:Standup\r\nEND:VEVENT\r\n\
+            BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\
+            RECURRENCE-ID:20260102T090000Z\r\nDTSTART:20260102T093000Z\r\nDTEND:20260102T103000Z\r\n\
+            SUMMARY:Standup (moved)\r\nEND:VEVENT\r\n\
+            END:VCALENDAR\r\n";
+        let entries = ical_to_jscalendar_entries(ical).expect("parse");
+        assert_eq!(entries.len(), 1, "calcard merges overrides into master");
+        let master = entries[0].data.as_object().unwrap();
+        assert!(master.contains_key("recurrenceOverrides"));
+    }
+
+    #[test]
+    fn strip_extracted_fields_pulls_isdraft_use_default_alerts() {
+        let mut v = serde_json::json!({
+            "@type": "Event",
+            "uid": "x",
+            "title": "T",
+            "isDraft": true,
+            "useDefaultAlerts": true,
+            "calendarIds": {"a": true},
+            "utcStart": "2026-01-01T09:00:00Z",
+            "method": "PUBLISH"
+        });
+        let (d, u, uid) = strip_extracted_fields_from_event(&mut v);
+        assert!(d);
+        assert!(u);
+        assert_eq!(uid.as_deref(), Some("x"));
+        let m = v.as_object().unwrap();
+        assert!(!m.contains_key("calendarIds"));
+        assert!(!m.contains_key("isDraft"));
+        assert!(!m.contains_key("useDefaultAlerts"));
+        assert!(!m.contains_key("method"));
+        assert!(!m.contains_key("utcStart"));
+    }
+}

src/sync/import_dav/collections.rs

@@ -0,0 +1,416 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use rusqlite::{Connection, params};
+
+use crate::dav::discover::DiscoveredCollection;
+use crate::dav::href::{Href, last_path_component};
+use crate::db::dav_ids;
+use crate::error::Error;
+use crate::logging::{LEVEL_PROGRESS, Logger};
+use crate::sync::TypeCounts;
+
+pub fn reconcile_calendars(
+    conn: &mut Connection,
+    source_id: i64,
+    discovered: &[DiscoveredCollection],
+    counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<Vec<(String, i64)>, Error> {
+    reconcile_calendar_like(
+        conn,
+        source_id,
+        discovered,
+        &RowOps {
+            type_name: dav_ids::CALENDAR,
+            upsert: upsert_calendar_row,
+            delete: delete_calendar_row,
+        },
+        counts,
+        logger,
+    )
+}
+
+pub fn reconcile_address_books(
+    conn: &mut Connection,
+    source_id: i64,
+    discovered: &[DiscoveredCollection],
+    counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<Vec<(String, i64)>, Error> {
+    reconcile_calendar_like(
+        conn,
+        source_id,
+        discovered,
+        &RowOps {
+            type_name: dav_ids::ADDRESS_BOOK,
+            upsert: upsert_address_book_row,
+            delete: delete_address_book_row,
+        },
+        counts,
+        logger,
+    )
+}
+
+type RowUpsert =
+    fn(&Connection, &DiscoveredCollection, Option<i64>) -> Result<i64, rusqlite::Error>;
+type RowDelete = fn(&Connection, i64) -> Result<(), rusqlite::Error>;
+
+struct RowOps {
+    type_name: &'static str,
+    upsert: RowUpsert,
+    delete: RowDelete,
+}
+
+fn reconcile_calendar_like(
+    conn: &mut Connection,
+    source_id: i64,
+    discovered: &[DiscoveredCollection],
+    ops: &RowOps,
+    counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<Vec<(String, i64)>, Error> {
+    let type_name = ops.type_name;
+    let upsert_row = ops.upsert;
+    let delete_row = ops.delete;
+    let existing: HashMap<String, i64> = dav_ids::collections_of_type(conn, source_id, type_name)
+        .map_err(|e| Error::Partial(e.to_string()))?;
+
+    let mut upserted: Vec<(String, i64)> = Vec::new();
+    let server_hrefs: std::collections::HashSet<String> = discovered
+        .iter()
+        .map(|c| c.href.as_str().to_owned())
+        .collect();
+
+    for coll in discovered {
+        let collection_href = coll.href.as_str().to_owned();
+        let existing_local = existing.get(&collection_href).copied();
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        match upsert_row(&tx, coll, existing_local) {
+            Ok(local_id) => {
+                if existing_local.is_none() {
+                    dav_ids::insert(
+                        &tx,
+                        source_id,
+                        type_name,
+                        &collection_href,
+                        &collection_href,
+                        "",
+                        local_id,
+                    )
+                    .map_err(|e| Error::Partial(e.to_string()))?;
+                    counts.created += 1;
+                } else {
+                    counts.fetched += 1;
+                }
+                tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                upserted.push((collection_href.clone(), local_id));
+                if logger.enabled(LEVEL_PROGRESS) {
+                    eprintln!("collection upserted: {collection_href} -> {local_id}");
+                }
+            }
+            Err(e) => {
+                let _ = tx.rollback();
+                logger.warn(&format!(
+                    "collection upsert {collection_href:?} failed: {e}"
+                ));
+                counts.failed += 1;
+            }
+        }
+    }
+
+    for (collection_href, local_id) in &existing {
+        if server_hrefs.contains(collection_href) {
+            continue;
+        }
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        let item_type = item_type_for(type_name);
+        if let Some(item_type) = item_type {
+            let _ = dav_ids::delete_collection(&tx, source_id, item_type, collection_href);
+        }
+        let _ = dav_ids::delete_collection(&tx, source_id, type_name, collection_href);
+        match delete_row(&tx, *local_id) {
+            Ok(()) => {
+                counts.deleted += 1;
+            }
+            Err(e) => {
+                logger.warn(&format!(
+                    "vanished collection {collection_href:?} delete failed: {e}"
+                ));
+                counts.failed += 1;
+            }
+        }
+        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    }
+
+    Ok(upserted)
+}
+
+fn item_type_for(collection_type: &str) -> Option<&'static str> {
+    match collection_type {
+        dav_ids::CALENDAR => Some(dav_ids::CALENDAR_EVENT),
+        dav_ids::ADDRESS_BOOK => Some(dav_ids::CONTACT_CARD),
+        _ => None,
+    }
+}
+
+fn upsert_calendar_row(
+    conn: &Connection,
+    coll: &DiscoveredCollection,
+    existing_local: Option<i64>,
+) -> Result<i64, rusqlite::Error> {
+    let name = display_or_fallback(&coll.props.displayname, &coll.href);
+    let description = coll.props.calendar_description.as_deref();
+    let color = coll.props.calendar_color.as_deref();
+    let sort_order = coll.props.calendar_order.unwrap_or(0).max(0);
+    let time_zone = parse_tzid_from_vtimezone(coll.props.calendar_timezone.as_deref());
+
+    if let Some(local) = existing_local {
+        conn.execute(
+            "UPDATE calendars SET name = ?1, description = ?2, color = ?3, sort_order = ?4,
+                                  time_zone = ?5
+             WHERE id = ?6",
+            params![name, description, color, sort_order, time_zone, local],
+        )?;
+        Ok(local)
+    } else {
+        conn.execute(
+            "INSERT INTO calendars (name, description, color, sort_order, is_subscribed,
+                                    is_visible, is_default, include_in_availability,
+                                    default_alerts_with_time, default_alerts_without_time, time_zone)
+             VALUES (?1, ?2, ?3, ?4, 1, 1, 0, 'all', NULL, NULL, ?5)",
+            params![name, description, color, sort_order, time_zone],
+        )?;
+        Ok(conn.last_insert_rowid())
+    }
+}
+
+fn delete_calendar_row(conn: &Connection, local_id: i64) -> Result<(), rusqlite::Error> {
+    conn.execute(
+        "DELETE FROM calendar_events
+         WHERE EXISTS (
+           SELECT 1 FROM json_each(calendar_ids) WHERE value = ?1
+         )",
+        params![local_id],
+    )?;
+    conn.execute("DELETE FROM calendars WHERE id = ?1", params![local_id])?;
+    Ok(())
+}
+
+fn upsert_address_book_row(
+    conn: &Connection,
+    coll: &DiscoveredCollection,
+    existing_local: Option<i64>,
+) -> Result<i64, rusqlite::Error> {
+    let name = display_or_fallback(&coll.props.displayname, &coll.href);
+    let description = coll.props.addressbook_description.as_deref();
+
+    if let Some(local) = existing_local {
+        conn.execute(
+            "UPDATE address_books SET name = ?1, description = ?2 WHERE id = ?3",
+            params![name, description, local],
+        )?;
+        Ok(local)
+    } else {
+        conn.execute(
+            "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed)
+             VALUES (?1, ?2, 0, 0, 1)",
+            params![name, description],
+        )?;
+        Ok(conn.last_insert_rowid())
+    }
+}
+
+fn delete_address_book_row(conn: &Connection, local_id: i64) -> Result<(), rusqlite::Error> {
+    conn.execute(
+        "DELETE FROM contact_cards
+         WHERE EXISTS (
+           SELECT 1 FROM json_each(address_book_ids) WHERE value = ?1
+         )",
+        params![local_id],
+    )?;
+    conn.execute("DELETE FROM address_books WHERE id = ?1", params![local_id])?;
+    Ok(())
+}
+
+fn display_or_fallback(name: &Option<String>, href: &Href) -> String {
+    if let Some(n) = name
+        && !n.trim().is_empty()
+    {
+        return n.clone();
+    }
+    let comp = last_path_component(href);
+    if comp.is_empty() {
+        "Untitled".to_owned()
+    } else {
+        comp
+    }
+}
+
+fn parse_tzid_from_vtimezone(vt: Option<&str>) -> Option<String> {
+    let body = vt?;
+    for line in body.lines() {
+        let line = line.trim();
+        if let Some(rest) = line.strip_prefix("TZID:") {
+            let v = rest.trim();
+            if !v.is_empty() {
+                return Some(v.to_owned());
+            }
+        }
+    }
+    None
+}
+
+pub fn upsert_root_directory(
+    conn: &Connection,
+    source_id: i64,
+    root: &DiscoveredCollection,
+) -> Result<i64, Error> {
+    let collection_href = root.href.as_str().to_owned();
+    let name = display_or_fallback(&root.props.displayname, &root.href);
+    if let Some(local) =
+        dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &collection_href)
+            .map_err(|e| Error::Partial(e.to_string()))?
+    {
+        conn.execute(
+            "UPDATE file_nodes SET name = ?1 WHERE id = ?2",
+            params![name, local],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        return Ok(local);
+    }
+    let now = time::OffsetDateTime::now_utc()
+        .format(&time::format_description::well_known::Rfc3339)
+        .map_err(|e| Error::Partial(format!("clock: {e}")))?;
+    conn.execute(
+        "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type,
+                                  created, modified, is_subscribed, role)
+         VALUES (NULL, 'directory', NULL, NULL, ?1, NULL, ?2, NULL, 1, NULL)",
+        params![name, now],
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let local_id = conn.last_insert_rowid();
+    dav_ids::insert(
+        conn,
+        source_id,
+        dav_ids::FILE_NODE,
+        &collection_href,
+        &collection_href,
+        "",
+        local_id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(local_id)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_tzid_from_vtimezone_extracts_first() {
+        let v = "BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nEND:VTIMEZONE\r\n";
+        assert_eq!(
+            parse_tzid_from_vtimezone(Some(v)).as_deref(),
+            Some("Europe/Berlin")
+        );
+    }
+
+    #[test]
+    fn parse_tzid_returns_none_when_absent() {
+        assert!(parse_tzid_from_vtimezone(Some("nothing here")).is_none());
+        assert!(parse_tzid_from_vtimezone(None).is_none());
+    }
+
+    #[test]
+    fn display_or_fallback_uses_href_when_displayname_empty() {
+        let href = Href::from_normalised("/dav/cal/u/work/".to_owned());
+        let name: Option<String> = None;
+        assert_eq!(display_or_fallback(&name, &href), "work");
+    }
+
+    #[test]
+    fn display_or_fallback_uses_displayname_when_present() {
+        let href = Href::from_normalised("/dav/cal/u/default/".to_owned());
+        let name = Some("Default".to_owned());
+        assert_eq!(display_or_fallback(&name, &href), "Default");
+    }
+
+    #[test]
+    fn delete_calendar_row_cascades_events_and_sync_id_rows() {
+        use crate::db::init;
+        use crate::db::sources::{SourceKey, upsert_source};
+        let conn = rusqlite::Connection::open_in_memory().unwrap();
+        init::apply_schema(&conn).unwrap();
+        let sid = upsert_source(
+            &conn,
+            &SourceKey {
+                kind: "caldav".to_owned(),
+                session_url: "https://x".to_owned(),
+                account_id: "u".to_owned(),
+            },
+            Some("u"),
+            "u",
+        )
+        .unwrap();
+        conn.execute(
+            "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible, is_default,
+                                     include_in_availability)
+             VALUES ('Work', 0, 1, 1, 0, 'all')",
+            [],
+        )
+        .unwrap();
+        let cal_local = conn.last_insert_rowid();
+        conn.execute(
+            "INSERT INTO calendar_events (calendar_ids, data, data_type)
+             VALUES (?1, '{}', 'Event')",
+            rusqlite::params![format!("[{cal_local}]")],
+        )
+        .unwrap();
+        let evt_local = conn.last_insert_rowid();
+        dav_ids::insert(
+            &conn,
+            sid,
+            dav_ids::CALENDAR,
+            "/cal/work/",
+            "/cal/work/",
+            "",
+            cal_local,
+        )
+        .unwrap();
+        dav_ids::insert(
+            &conn,
+            sid,
+            dav_ids::CALENDAR_EVENT,
+            "/cal/work/",
+            "/cal/work/e.ics",
+            "\"v\"",
+            evt_local,
+        )
+        .unwrap();
+        let _ = dav_ids::delete_collection(&conn, sid, dav_ids::CALENDAR_EVENT, "/cal/work/");
+        let _ = dav_ids::delete_collection(&conn, sid, dav_ids::CALENDAR, "/cal/work/");
+        delete_calendar_row(&conn, cal_local).unwrap();
+        let cal_count: i64 = conn
+            .query_row("SELECT count(*) FROM calendars", [], |r| r.get(0))
+            .unwrap();
+        let evt_count: i64 = conn
+            .query_row("SELECT count(*) FROM calendar_events", [], |r| r.get(0))
+            .unwrap();
+        let dav_count: i64 = conn
+            .query_row("SELECT count(*) FROM sync_id_dav", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(cal_count, 0, "calendar row gone");
+        assert_eq!(evt_count, 0, "event rows cascaded");
+        assert_eq!(dav_count, 0, "sync_id_dav rows for both types cleared");
+    }
+}

src/sync/import_dav/coordinator.rs

@@ -0,0 +1,520 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::Connection;
+
+use crate::dav::client::DavClient;
+use crate::dav::discover::{DavKind, Discovery, DiscoveryError, discover};
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::error::Error;
+use crate::jmap::http::{Auth, RetryPolicy};
+use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+
+use super::collections;
+use super::items;
+use super::tree;
+
+#[derive(Debug, Clone, Copy)]
+pub enum DavKindArg {
+    Caldav,
+    Carddav,
+    Webdav,
+}
+
+impl DavKindArg {
+    pub fn kind(self) -> DavKind {
+        match self {
+            DavKindArg::Caldav => DavKind::Caldav,
+            DavKindArg::Carddav => DavKind::Carddav,
+            DavKindArg::Webdav => DavKind::Webdav,
+        }
+    }
+
+    pub fn source_kind(self) -> &'static str {
+        match self {
+            DavKindArg::Caldav => "caldav",
+            DavKindArg::Carddav => "carddav",
+            DavKindArg::Webdav => "webdav",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum DavAuth {
+    Basic { user: String, password: String },
+    Bearer { token: String },
+}
+
+impl DavAuth {
+    pub fn to_jmap_auth(&self) -> Auth {
+        match self {
+            DavAuth::Basic { user, password } => Auth::Basic {
+                user: user.clone(),
+                password: password.clone(),
+            },
+            DavAuth::Bearer { token } => Auth::Bearer {
+                token: token.clone(),
+            },
+        }
+    }
+
+    pub fn username(&self) -> String {
+        match self {
+            DavAuth::Basic { user, .. } => user.clone(),
+            DavAuth::Bearer { .. } => "(bearer)".to_owned(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct DavImportConfig {
+    pub kind: DavKindArg,
+    pub url: String,
+    pub auth: DavAuth,
+    pub allow_cleartext: bool,
+    pub dav_connections: usize,
+    pub multiget_batch: usize,
+    pub allow_source_change: bool,
+}
+
+pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    enforce_tls_policy(&config.url, config.allow_cleartext)?;
+
+    let mut conn = db::init::open(&common.archive)?;
+
+    let client = DavClient::new(
+        config.auth.to_jmap_auth(),
+        RetryPolicy::new(common.max_retries),
+        common.allow_invalid_certs,
+    );
+    client.set_logger(logger);
+
+    let discovery =
+        discover(&client, config.kind.kind(), &config.url).map_err(map_discovery_error)?;
+
+    let session_url = normalise_base_url(&config.url);
+    let account_id_raw = discovery
+        .principal_url
+        .clone()
+        .unwrap_or_else(|| discovery.home_set_url.clone());
+    let account_id = normalise_account_url(&account_id_raw);
+
+    let key = SourceKey {
+        kind: config.kind.source_kind().to_owned(),
+        session_url: session_url.clone(),
+        account_id: account_id.clone(),
+    };
+
+    if !common.dry_run
+        && let Some((url, acc)) = db::sources::conflicting_source(
+            &conn,
+            config.kind.source_kind(),
+            &session_url,
+            &account_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records {} source ({url}, account {acc}); \
+             pass --allow-source-change to import a different account",
+            config.kind.source_kind()
+        )));
+    }
+
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "DAV discovery: principal={:?} home_set={} collections={}",
+            discovery.principal_url,
+            discovery.home_set_url,
+            discovery.collections.len()
+        );
+    }
+
+    if common.dry_run {
+        let mut summary = run_dry_diff(&conn, &client, &discovery, &config, logger)?;
+        summary.retries_observed = client.retries_observed();
+        summary.retry_after_sleeps = client.retry_after_sleeps();
+        return Ok(summary);
+    }
+
+    let username = config.auth.username();
+    let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username)
+        .map_err(|e| Error::Partial(e.to_string()))?;
+
+    let summary = match config.kind {
+        DavKindArg::Caldav => {
+            run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)?
+        }
+        DavKindArg::Carddav => {
+            run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)?
+        }
+        DavKindArg::Webdav => {
+            run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)?
+        }
+    };
+
+    if !summary.any_failed()
+        && let Err(e) = run_gc(&conn)
+    {
+        logger.warn(&format!("blob GC skipped: {e}"));
+    }
+
+    let mut summary = summary;
+    summary.retries_observed = client.retries_observed();
+    summary.retry_after_sleeps = client.retry_after_sleeps();
+    Ok(summary)
+}
+
+type ReconcileCollections = fn(
+    &mut Connection,
+    i64,
+    &[crate::dav::discover::DiscoveredCollection],
+    &mut TypeCounts,
+    Logger,
+) -> Result<Vec<(String, i64)>, Error>;
+
+type ReconcileItems =
+    fn(&mut Connection, &items::ItemRunCtx<'_>, &str, i64, &mut TypeCounts) -> Result<(), Error>;
+
+struct ItemPhase {
+    container_label: &'static str,
+    item_label: &'static str,
+    reconcile_collections: ReconcileCollections,
+    reconcile_items: ReconcileItems,
+}
+
+fn run_collection_phase(
+    conn: &mut Connection,
+    client: &DavClient,
+    source_id: i64,
+    discovery: &Discovery,
+    config: &DavImportConfig,
+    logger: Logger,
+    phase: ItemPhase,
+) -> Result<Summary, Error> {
+    let mut summary = Summary::default();
+    let mut container_counts = TypeCounts::default();
+    let mut item_counts = TypeCounts::default();
+
+    let upserted = (phase.reconcile_collections)(
+        conn,
+        source_id,
+        &discovery.collections,
+        &mut container_counts,
+        logger,
+    )?;
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "import: {} done (upserted={} deleted={} failed={})",
+            phase.container_label,
+            container_counts.created + container_counts.fetched,
+            container_counts.deleted,
+            container_counts.failed
+        );
+    }
+
+    let ctx = items::ItemRunCtx {
+        client,
+        source_id,
+        base_url: &discovery.home_set_url,
+        multiget_batch: config.multiget_batch,
+        dav_connections: config.dav_connections,
+        logger,
+    };
+    for (collection_href, local_id) in &upserted {
+        match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut item_counts) {
+            Ok(()) => {}
+            Err(e) => {
+                logger.warn(&format!(
+                    "{} {collection_href:?}: items failed: {e}",
+                    phase.container_label
+                ));
+                item_counts.failed += 1;
+            }
+        }
+    }
+
+    summary
+        .per_type
+        .push((phase.container_label, container_counts));
+    summary.per_type.push((phase.item_label, item_counts));
+    Ok(summary)
+}
+
+fn run_dry_diff(
+    conn: &rusqlite::Connection,
+    client: &DavClient,
+    discovery: &Discovery,
+    config: &DavImportConfig,
+    logger: Logger,
+) -> Result<Summary, Error> {
+    use crate::dav::href::join_absolute;
+    use crate::dav::xml;
+    use crate::db::dav_ids;
+    let mut summary = Summary::default();
+    let (container_label, item_label, container_type, item_type) = match config.kind {
+        DavKindArg::Caldav => (
+            "calendar",
+            "calendarevent",
+            dav_ids::CALENDAR,
+            dav_ids::CALENDAR_EVENT,
+        ),
+        DavKindArg::Carddav => (
+            "addressbook",
+            "contactcard",
+            dav_ids::ADDRESS_BOOK,
+            dav_ids::CONTACT_CARD,
+        ),
+        DavKindArg::Webdav => (
+            "filenode",
+            "filenode",
+            dav_ids::FILE_NODE,
+            dav_ids::FILE_NODE,
+        ),
+    };
+
+    let mut container_counts = TypeCounts::default();
+    let mut item_counts = TypeCounts::default();
+    container_counts.created = discovery.collections.len() as u64;
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "dry-run: {} new={} (no archive to diff against)",
+            container_label,
+            discovery.collections.len(),
+        );
+    }
+
+    if !matches!(config.kind, DavKindArg::Webdav) {
+        for coll in &discovery.collections {
+            let url = join_absolute(&discovery.home_set_url, coll.href.as_str())
+                .map_err(|e| Error::Partial(e.to_string()))?;
+            let ms = client
+                .propfind_responses(&url, 1, &xml::propfind_dav_items(), &url)
+                .map_err(Error::from)?;
+            let new_count = ms
+                .responses
+                .iter()
+                .filter(|r| !r.props.is_collection)
+                .count();
+            item_counts.created += new_count as u64;
+            if logger.enabled(LEVEL_DEFAULT) {
+                eprintln!("  {} items: {new_count}", coll.href.as_str());
+            }
+        }
+    } else if let Some(root) = discovery.collections.first() {
+        let url = join_absolute(&discovery.home_set_url, root.href.as_str())
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        let ms = client
+            .propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url)
+            .map_err(Error::from)?;
+        let new_count = ms
+            .responses
+            .iter()
+            .filter(|r| !r.props.is_collection)
+            .count();
+        item_counts.created += new_count as u64;
+        if logger.enabled(LEVEL_DEFAULT) {
+            eprintln!("  root {} files: {new_count}", root.href.as_str());
+        }
+    }
+
+    let _ = container_type;
+    let _ = item_type;
+    let _ = conn;
+    summary.per_type.push((container_label, container_counts));
+    if !matches!(config.kind, DavKindArg::Webdav) {
+        summary.per_type.push((item_label, item_counts));
+    }
+    Ok(summary)
+}
+
+fn run_caldav(
+    conn: &mut Connection,
+    client: &DavClient,
+    source_id: i64,
+    discovery: &Discovery,
+    config: &DavImportConfig,
+    logger: Logger,
+) -> Result<Summary, Error> {
+    run_collection_phase(
+        conn,
+        client,
+        source_id,
+        discovery,
+        config,
+        logger,
+        ItemPhase {
+            container_label: "calendar",
+            item_label: "calendarevent",
+            reconcile_collections: collections::reconcile_calendars,
+            reconcile_items: items::reconcile_calendar_events,
+        },
+    )
+}
+
+fn run_carddav(
+    conn: &mut Connection,
+    client: &DavClient,
+    source_id: i64,
+    discovery: &Discovery,
+    config: &DavImportConfig,
+    logger: Logger,
+) -> Result<Summary, Error> {
+    run_collection_phase(
+        conn,
+        client,
+        source_id,
+        discovery,
+        config,
+        logger,
+        ItemPhase {
+            container_label: "addressbook",
+            item_label: "contactcard",
+            reconcile_collections: collections::reconcile_address_books,
+            reconcile_items: items::reconcile_contact_cards,
+        },
+    )
+}
+
+fn run_webdav(
+    conn: &mut Connection,
+    client: &DavClient,
+    source_id: i64,
+    discovery: &Discovery,
+    config: &DavImportConfig,
+    logger: Logger,
+) -> Result<Summary, Error> {
+    let mut summary = Summary::default();
+    let mut file_counts = TypeCounts::default();
+
+    if discovery.collections.is_empty() {
+        summary.per_type.push(("filenode", file_counts));
+        return Ok(summary);
+    }
+    let root = &discovery.collections[0];
+    let ctx = tree::WebDavCtx {
+        client,
+        source_id,
+        base_url: &discovery.home_set_url,
+        dav_connections: config.dav_connections,
+        logger,
+    };
+    tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts)?;
+
+    summary.per_type.push(("filenode", file_counts));
+    Ok(summary)
+}
+
+fn map_discovery_error(err: DiscoveryError) -> Error {
+    match err {
+        DiscoveryError::NotFound { url } => Error::Usage(format!(
+            "no DAV collections found under --url {url:?}; \
+             check the URL and that the account has access"
+        )),
+        DiscoveryError::Transport(e) => Error::from(e),
+        DiscoveryError::Parse(e) => {
+            Error::Connection(format!("DAV discovery: malformed multistatus: {e}"))
+        }
+        DiscoveryError::Href(e) => Error::Connection(format!("DAV discovery: bad href: {e}")),
+    }
+}
+
+fn run_gc(conn: &Connection) -> Result<(), Error> {
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    db::blobs::gc_orphan_blobs(&tx).map_err(|e| Error::Partial(e.to_string()))?;
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}
+
+fn enforce_tls_policy(url: &str, allow_cleartext: bool) -> Result<(), Error> {
+    let parsed =
+        url::Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?;
+    match parsed.scheme() {
+        "https" => Ok(()),
+        "http" => {
+            if allow_cleartext {
+                Ok(())
+            } else {
+                Err(Error::Connection(
+                    "http:// URL requires --allow-cleartext".to_owned(),
+                ))
+            }
+        }
+        other => Err(Error::Usage(format!(
+            "--url scheme must be http or https, got {other}"
+        ))),
+    }
+}
+
+fn normalise_base_url(url: &str) -> String {
+    let parsed = match url::Url::parse(url) {
+        Ok(u) => u,
+        Err(_) => return url.to_owned(),
+    };
+    let scheme = parsed.scheme();
+    let host = parsed.host_str().unwrap_or("");
+    let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default();
+    format!("{scheme}://{host}{port}")
+}
+
+fn normalise_account_url(url: &str) -> String {
+    let mut parsed = match url::Url::parse(url) {
+        Ok(u) => u,
+        Err(_) => return url.to_owned(),
+    };
+    parsed.set_query(None);
+    parsed.set_fragment(None);
+    let path = parsed.path().to_owned();
+    if !path.ends_with('/') {
+        parsed.set_path(&format!("{path}/"));
+    }
+    parsed.to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn https_url_is_accepted_without_allow_cleartext() {
+        assert!(enforce_tls_policy("https://x/", false).is_ok());
+    }
+
+    #[test]
+    fn http_url_requires_allow_cleartext() {
+        let err = enforce_tls_policy("http://x/", false).unwrap_err();
+        assert!(matches!(err, Error::Connection(_)));
+        assert!(enforce_tls_policy("http://x/", true).is_ok());
+    }
+
+    #[test]
+    fn ftp_url_rejected_as_usage() {
+        let err = enforce_tls_policy("ftp://x/", false).unwrap_err();
+        assert!(matches!(err, Error::Usage(_)));
+    }
+
+    #[test]
+    fn normalise_base_url_strips_path() {
+        assert_eq!(
+            normalise_base_url("https://dav.example.com/cal/"),
+            "https://dav.example.com"
+        );
+    }
+
+    #[test]
+    fn normalise_base_url_keeps_explicit_port() {
+        assert_eq!(
+            normalise_base_url("https://dav.example.com:8443/cal/"),
+            "https://dav.example.com:8443"
+        );
+    }
+}

src/sync/import_dav/items.rs

@@ -0,0 +1,653 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+
+use rusqlite::{Connection, params};
+
+use crate::dav::client::{DavClient, MultiStatus};
+use crate::dav::href::{Href, join_absolute, normalise};
+use crate::dav::parse::DavResponse;
+use crate::dav::xml;
+use crate::db::dav_ids;
+use crate::error::Error;
+use crate::jmap::error::JmapError;
+use crate::logging::{LEVEL_PROGRESS, Logger};
+use crate::sync::TypeCounts;
+use crate::sync::import_jmap::pool::Pool;
+
+use super::calcard;
+
+pub struct ItemRunCtx<'a> {
+    pub client: &'a DavClient,
+    pub source_id: i64,
+    pub base_url: &'a str,
+    pub multiget_batch: usize,
+    pub dav_connections: usize,
+    pub logger: Logger,
+}
+
+struct PerCollectionCtx<'a> {
+    run: &'a ItemRunCtx<'a>,
+    kind: ItemKind,
+    collection_href: &'a str,
+    container_local_id: i64,
+    absolute_url: &'a str,
+}
+
+pub fn reconcile_calendar_events(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    collection_href: &str,
+    calendar_local_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let absolute = absolute_collection_url(ctx.base_url, collection_href)?;
+    let server_items = enumerate_items(ctx.client, &absolute)?;
+    let pcc = PerCollectionCtx {
+        run: ctx,
+        kind: ItemKind::CalendarEvent,
+        collection_href,
+        container_local_id: calendar_local_id,
+        absolute_url: &absolute,
+    };
+    reconcile_items_generic(conn, &pcc, &server_items, counts)
+}
+
+pub fn reconcile_contact_cards(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    collection_href: &str,
+    address_book_local_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let absolute = absolute_collection_url(ctx.base_url, collection_href)?;
+    let server_items = enumerate_items(ctx.client, &absolute)?;
+    let pcc = PerCollectionCtx {
+        run: ctx,
+        kind: ItemKind::ContactCard,
+        collection_href,
+        container_local_id: address_book_local_id,
+        absolute_url: &absolute,
+    };
+    reconcile_items_generic(conn, &pcc, &server_items, counts)
+}
+
+#[derive(Debug, Clone, Copy)]
+enum ItemKind {
+    CalendarEvent,
+    ContactCard,
+}
+
+impl ItemKind {
+    fn type_name(self) -> &'static str {
+        match self {
+            ItemKind::CalendarEvent => dav_ids::CALENDAR_EVENT,
+            ItemKind::ContactCard => dav_ids::CONTACT_CARD,
+        }
+    }
+    fn data_query(self) -> fn(&[Href]) -> String {
+        match self {
+            ItemKind::CalendarEvent => xml::calendar_multiget,
+            ItemKind::ContactCard => xml::addressbook_multiget,
+        }
+    }
+    fn data_field(self) -> &'static str {
+        match self {
+            ItemKind::CalendarEvent => "calendar_data",
+            ItemKind::ContactCard => "address_data",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct ServerItem {
+    href: String,
+    etag: String,
+}
+
+fn enumerate_items(client: &DavClient, url: &str) -> Result<Vec<ServerItem>, Error> {
+    let body = xml::propfind_dav_items();
+    let ms = client
+        .propfind_responses(url, 1, &body, url)
+        .map_err(Error::from)?;
+    if ms.status >= 400 {
+        return Err(Error::Partial(format!(
+            "enumerate {url}: http {}",
+            ms.status
+        )));
+    }
+    let self_href = normalise(url, "")
+        .map(|h| h.into_string())
+        .unwrap_or_default();
+    let mut out = Vec::new();
+    for r in ms.responses {
+        if r.href.as_str() == self_href {
+            continue;
+        }
+        if r.props.is_collection {
+            continue;
+        }
+        out.push(ServerItem {
+            href: r.href.into_string(),
+            etag: r.props.etag.unwrap_or_default(),
+        });
+    }
+    Ok(out)
+}
+
+struct MultigetJob {
+    body: String,
+    hrefs: Vec<String>,
+}
+
+struct MultigetReply {
+    hrefs: Vec<String>,
+    result: Result<MultiStatus, JmapError>,
+}
+
+struct GetJob {
+    url: String,
+    item: ServerItem,
+}
+
+struct GetReply {
+    item: ServerItem,
+    result: Result<GetBody, JmapError>,
+}
+
+struct GetBody {
+    status: u16,
+    bytes: Vec<u8>,
+    etag: Option<String>,
+}
+
+fn reconcile_items_generic(
+    conn: &mut Connection,
+    pcc: &PerCollectionCtx<'_>,
+    server_items: &[ServerItem],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let type_name = pcc.kind.type_name();
+    let local_rows =
+        dav_ids::items_in_collection(conn, pcc.run.source_id, type_name, pcc.collection_href)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let local_map: HashMap<String, (String, i64)> = local_rows
+        .iter()
+        .map(|r| (r.item_href.clone(), (r.etag.clone(), r.local_id)))
+        .collect();
+    let server_set: HashSet<&str> = server_items.iter().map(|s| s.href.as_str()).collect();
+
+    let mut to_fetch: Vec<ServerItem> = Vec::new();
+    for s in server_items {
+        match local_map.get(&s.href) {
+            None => to_fetch.push(s.clone()),
+            Some((local_etag, _)) => {
+                if s.etag.is_empty() || s.etag != *local_etag {
+                    to_fetch.push(s.clone());
+                }
+            }
+        }
+    }
+
+    let vanished: Vec<String> = local_map
+        .keys()
+        .filter(|h| !server_set.contains(h.as_str()))
+        .cloned()
+        .collect();
+
+    if !to_fetch.is_empty() {
+        let fallback = run_multiget_pool(conn, pcc, &to_fetch, &local_map, counts)?;
+        if !fallback.is_empty() {
+            run_get_pool(conn, pcc, &fallback, &local_map, counts)?;
+        }
+    }
+
+    for href in &vanished {
+        let local_id = match local_map.get(href) {
+            Some((_, id)) => *id,
+            None => continue,
+        };
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        let table = match pcc.kind {
+            ItemKind::CalendarEvent => "calendar_events",
+            ItemKind::ContactCard => "contact_cards",
+        };
+        tx.execute(
+            &format!("DELETE FROM {table} WHERE id = ?1"),
+            params![local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        dav_ids::delete_item(&tx, pcc.run.source_id, type_name, href)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+        counts.deleted += 1;
+    }
+
+    if pcc.run.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "items in {}: fetched={} deleted={} failed={}",
+            pcc.collection_href, counts.fetched, counts.deleted, counts.failed
+        );
+    }
+    Ok(())
+}
+
+fn run_multiget_pool(
+    conn: &mut Connection,
+    pcc: &PerCollectionCtx<'_>,
+    to_fetch: &[ServerItem],
+    local_map: &HashMap<String, (String, i64)>,
+    counts: &mut TypeCounts,
+) -> Result<Vec<ServerItem>, Error> {
+    let logger = pcc.run.logger;
+    let absolute_url = pcc.absolute_url;
+    let batch_size = pcc.run.multiget_batch.max(1);
+    let workers = pcc.run.dav_connections.clamp(1, 8);
+    let by_href: HashMap<String, ServerItem> = to_fetch
+        .iter()
+        .map(|s| (s.href.clone(), s.clone()))
+        .collect();
+    let client_for_pool = pcc.run.client.clone();
+    let url_for_pool = absolute_url.to_owned();
+    let pool: Pool<MultigetJob, MultigetReply> = Pool::new(workers, move |job: MultigetJob| {
+        let result = client_for_pool.report_responses(&url_for_pool, 1, &job.body, &url_for_pool);
+        MultigetReply {
+            hrefs: job.hrefs,
+            result,
+        }
+    });
+    let mut submitted: usize = 0;
+    for chunk in to_fetch.chunks(batch_size) {
+        let hrefs: Vec<Href> = chunk
+            .iter()
+            .map(|s| Href::from_normalised(s.href.clone()))
+            .collect();
+        let chunk_hrefs: Vec<String> = chunk.iter().map(|s| s.href.clone()).collect();
+        pool.submit(MultigetJob {
+            body: (pcc.kind.data_query())(&hrefs),
+            hrefs: chunk_hrefs,
+        });
+        submitted += 1;
+    }
+
+    let mut fallback: Vec<ServerItem> = Vec::new();
+    for _ in 0..submitted {
+        let reply = match pool.results().recv() {
+            Ok(r) => r,
+            Err(_) => break,
+        };
+        match reply.result {
+            Err(JmapError::HttpStatus { status, .. }) if status == 405 || status == 501 => {
+                logger.warn(&format!(
+                    "multiget {absolute_url} returned {status}; server may not implement multiget, falling back to per-item GET"
+                ));
+                fallback.extend(reply.hrefs.iter().filter_map(|h| by_href.get(h).cloned()));
+            }
+            Err(e) => {
+                logger.warn(&format!(
+                    "multiget {absolute_url}: {e}; falling back to per-item GET"
+                ));
+                fallback.extend(reply.hrefs.iter().filter_map(|h| by_href.get(h).cloned()));
+            }
+            Ok(ms) if ms.status >= 400 => {
+                logger.warn(&format!(
+                    "multiget {absolute_url} returned {}; chunk failed",
+                    ms.status
+                ));
+                counts.failed += reply.hrefs.len() as u64;
+            }
+            Ok(ms) => {
+                let requested: HashSet<&str> = reply.hrefs.iter().map(String::as_str).collect();
+                let mut tx = conn
+                    .unchecked_transaction()
+                    .map_err(|e| Error::Partial(e.to_string()))?;
+                let mut pending: usize = 0;
+                for r in ms.responses {
+                    if !requested.contains(r.href.as_str()) {
+                        logger.warn(&format!(
+                            "multiget {absolute_url}: server returned unsolicited href {}; skipping",
+                            r.href.as_str()
+                        ));
+                        continue;
+                    }
+                    handle_multiget_response(&tx, pcc, &r, local_map, counts);
+                    pending += 1;
+                    if pending >= COMMIT_BATCH {
+                        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                        tx = conn
+                            .unchecked_transaction()
+                            .map_err(|e| Error::Partial(e.to_string()))?;
+                        pending = 0;
+                    }
+                }
+                tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+            }
+        }
+    }
+    drop(pool);
+    Ok(fallback)
+}
+
+fn run_get_pool(
+    conn: &mut Connection,
+    pcc: &PerCollectionCtx<'_>,
+    items: &[ServerItem],
+    local_map: &HashMap<String, (String, i64)>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let logger = pcc.run.logger;
+    let absolute_url = pcc.absolute_url;
+    let workers = pcc.run.dav_connections.clamp(1, 8);
+    let client_for_pool = pcc.run.client.clone();
+    let pool: Pool<GetJob, GetReply> = Pool::new(workers, move |job: GetJob| {
+        let r = client_for_pool.get(&job.url).map(|resp| GetBody {
+            status: resp.status,
+            bytes: resp.body,
+            etag: resp.etag,
+        });
+        GetReply {
+            item: job.item,
+            result: r,
+        }
+    });
+
+    let mut submitted: usize = 0;
+    for item in items {
+        let url = match absolute_item_url(absolute_url, &item.href) {
+            Ok(u) => u,
+            Err(e) => {
+                logger.warn(&format!("item url {}: {e}", item.href));
+                counts.failed += 1;
+                continue;
+            }
+        };
+        pool.submit(GetJob {
+            url,
+            item: item.clone(),
+        });
+        submitted += 1;
+    }
+
+    let mut tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let mut pending: usize = 0;
+    for _ in 0..submitted {
+        let reply = match pool.results().recv() {
+            Ok(r) => r,
+            Err(_) => break,
+        };
+        match reply.result {
+            Err(e) => {
+                logger.warn(&format!("GET {}: {e}", reply.item.href));
+                counts.failed += 1;
+            }
+            Ok(body) if body.status == 404 || body.status == 410 => {}
+            Ok(body) if body.status >= 400 => {
+                logger.warn(&format!("GET {}: http {}", reply.item.href, body.status));
+                counts.failed += 1;
+            }
+            Ok(body) => {
+                let etag = body.etag.unwrap_or_else(|| reply.item.etag.clone());
+                let raw = match std::str::from_utf8(&body.bytes) {
+                    Ok(s) => s.to_owned(),
+                    Err(e) => {
+                        logger.warn(&format!("GET {}: non-utf8 body: {e}", reply.item.href));
+                        counts.failed += 1;
+                        continue;
+                    }
+                };
+                if let Err(e) =
+                    insert_or_update_tx(&tx, pcc, &reply.item.href, &etag, &raw, local_map)
+                {
+                    logger.warn(&format!("item {}: {e}", reply.item.href));
+                    counts.failed += 1;
+                } else {
+                    counts.fetched += 1;
+                    pending += 1;
+                    if pending >= COMMIT_BATCH {
+                        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                        tx = conn
+                            .unchecked_transaction()
+                            .map_err(|e| Error::Partial(e.to_string()))?;
+                        pending = 0;
+                    }
+                }
+            }
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    drop(pool);
+    Ok(())
+}
+
+const COMMIT_BATCH: usize = 64;
+
+fn handle_multiget_response(
+    tx: &rusqlite::Transaction<'_>,
+    pcc: &PerCollectionCtx<'_>,
+    response: &DavResponse,
+    local_map: &HashMap<String, (String, i64)>,
+    counts: &mut TypeCounts,
+) {
+    let logger = pcc.run.logger;
+    if response_indicates_vanished(response) {
+        return;
+    }
+    let item_href = response.href.as_str().to_owned();
+    let etag = response.props.etag.clone().unwrap_or_default();
+    let data = match pcc.kind {
+        ItemKind::CalendarEvent => response.props.calendar_data.as_deref(),
+        ItemKind::ContactCard => response.props.address_data.as_deref(),
+    };
+    let Some(raw) = data else {
+        logger.warn(&format!(
+            "multiget {item_href}: missing {} payload",
+            pcc.kind.data_field()
+        ));
+        counts.failed += 1;
+        return;
+    };
+    if let Err(e) = insert_or_update_tx(tx, pcc, &item_href, &etag, raw, local_map) {
+        logger.warn(&format!("item {item_href}: {e}"));
+        counts.failed += 1;
+    } else {
+        counts.fetched += 1;
+    }
+}
+
+fn insert_or_update_tx(
+    tx: &rusqlite::Transaction<'_>,
+    pcc: &PerCollectionCtx<'_>,
+    item_href: &str,
+    etag: &str,
+    raw: &str,
+    local_map: &HashMap<String, (String, i64)>,
+) -> Result<(), Error> {
+    let existing = local_map.get(item_href).map(|(_, id)| *id);
+    match pcc.kind {
+        ItemKind::CalendarEvent => insert_or_update_event(
+            tx,
+            pcc.container_local_id,
+            existing,
+            raw,
+            item_href,
+            pcc.run.logger,
+        )?,
+        ItemKind::ContactCard => {
+            insert_or_update_card(tx, pcc.container_local_id, existing, raw, item_href)?
+        }
+    };
+    if existing.is_none() {
+        let new_local = tx.last_insert_rowid();
+        dav_ids::insert(
+            tx,
+            pcc.run.source_id,
+            pcc.kind.type_name(),
+            pcc.collection_href,
+            item_href,
+            etag,
+            new_local,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    } else {
+        dav_ids::update_etag(tx, pcc.run.source_id, pcc.kind.type_name(), item_href, etag)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn insert_or_update_event(
+    tx: &rusqlite::Transaction<'_>,
+    calendar_local: i64,
+    existing: Option<i64>,
+    raw: &str,
+    item_href: &str,
+    logger: Logger,
+) -> Result<(), Error> {
+    let entries = calcard::ical_to_jscalendar_entries(raw)
+        .map_err(|e| Error::Partial(format!("iCalendar parse: {e}")))?;
+    if entries.len() > 1 && logger.enabled(crate::logging::LEVEL_DEFAULT) {
+        logger.warn(&format!(
+            "{item_href}: iCalendar resource has {} entries; storing the first ({}), \
+             others dropped (recurrence overrides on the master are preserved)",
+            entries.len(),
+            entries[0].data_type.as_column(),
+        ));
+    }
+    let mut first = entries
+        .into_iter()
+        .next()
+        .ok_or_else(|| Error::Partial("iCalendar contained no parseable entries".to_owned()))?;
+    let (is_draft, use_default_alerts, _uid) =
+        calcard::strip_extracted_fields_from_event(&mut first.data);
+    let data_type = first.data_type.as_column();
+    let calendar_ids = format!("[{calendar_local}]");
+    if let Some(local) = existing {
+        tx.execute(
+            "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2,
+                                          use_default_alerts = ?3, data = ?4, data_type = ?5
+             WHERE id = ?6",
+            params![
+                calendar_ids,
+                is_draft as i64,
+                use_default_alerts as i64,
+                first.data.to_string(),
+                data_type,
+                local,
+            ],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    } else {
+        tx.execute(
+            "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type)
+             VALUES (?1, ?2, ?3, ?4, ?5)",
+            params![
+                calendar_ids,
+                is_draft as i64,
+                use_default_alerts as i64,
+                first.data.to_string(),
+                data_type,
+            ],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn insert_or_update_card(
+    tx: &rusqlite::Transaction<'_>,
+    address_book_local: i64,
+    existing: Option<i64>,
+    raw: &str,
+    item_href: &str,
+) -> Result<(), Error> {
+    let card = calcard::vcard_to_jscontact(raw, item_href)
+        .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?;
+    let address_book_ids = format!("[{address_book_local}]");
+    if let Some(local) = existing {
+        tx.execute(
+            "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3
+             WHERE id = ?4",
+            params![card.uid, address_book_ids, card.data.to_string(), local],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    } else {
+        tx.execute(
+            "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)",
+            params![card.uid, address_book_ids, card.data.to_string()],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn response_indicates_vanished(response: &DavResponse) -> bool {
+    if let Some(status) = response.status
+        && (status == 404 || status == 410)
+    {
+        return true;
+    }
+    response
+        .propstat_errors
+        .iter()
+        .any(|s| *s == 404 || *s == 410)
+}
+
+fn absolute_collection_url(base_url: &str, href: &str) -> Result<String, Error> {
+    join_absolute(base_url, href).map_err(|e| Error::Partial(e.to_string()))
+}
+
+fn absolute_item_url(collection_url: &str, item_href: &str) -> Result<String, Error> {
+    join_absolute(collection_url, item_href).map_err(|e| Error::Partial(e.to_string()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn absolute_collection_url_joins_path() {
+        let u = absolute_collection_url("https://x/dav/", "/dav/cal/u/d/").unwrap();
+        assert_eq!(u, "https://x/dav/cal/u/d/");
+    }
+
+    #[test]
+    fn absolute_item_url_joins_relative() {
+        let u = absolute_item_url("https://x/dav/cal/u/d/", "/dav/cal/u/d/event.ics").unwrap();
+        assert_eq!(u, "https://x/dav/cal/u/d/event.ics");
+    }
+
+    fn response(status: Option<u16>, propstat_errors: Vec<u16>) -> DavResponse {
+        DavResponse {
+            href: Href::from_normalised("/dav/cal/u/d/e.ics".to_owned()),
+            status,
+            props: crate::dav::parse::ResourceProps::default(),
+            propstat_errors,
+        }
+    }
+
+    #[test]
+    fn response_level_404_is_vanished() {
+        assert!(response_indicates_vanished(&response(Some(404), vec![])));
+        assert!(response_indicates_vanished(&response(Some(410), vec![])));
+    }
+
+    #[test]
+    fn per_propstat_404_or_410_is_vanished() {
+        assert!(response_indicates_vanished(&response(None, vec![404])));
+        assert!(response_indicates_vanished(&response(None, vec![200, 410])));
+    }
+
+    #[test]
+    fn other_propstat_errors_are_not_vanished() {
+        assert!(!response_indicates_vanished(&response(None, vec![403])));
+        assert!(!response_indicates_vanished(&response(None, vec![])));
+    }
+}

src/sync/import_dav/tree.rs

@@ -0,0 +1,681 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashSet, VecDeque};
+use std::io::Read;
+
+use rusqlite::{Connection, OptionalExtension, params};
+use time::format_description::well_known::Rfc3339;
+
+use crate::dav::client::DavClient;
+use crate::dav::discover::DiscoveredCollection;
+use crate::dav::href::{Href, join_absolute, last_path_component};
+use crate::dav::parse::DavResponse;
+use crate::dav::xml;
+use crate::db;
+use crate::db::dav_ids;
+use crate::error::Error;
+use crate::jmap::error::JmapError;
+use crate::logging::{LEVEL_PROGRESS, Logger};
+use crate::sync::TypeCounts;
+use crate::sync::import_jmap::pool::Pool;
+
+struct FilePlan {
+    item_href: String,
+    parent_href: String,
+    parent_local: i64,
+    existing_local: Option<i64>,
+    propfind_etag: String,
+    propfind_content_type: Option<String>,
+    propfind_last_modified: Option<String>,
+    propfind_creation_date: Option<String>,
+    propfind_displayname: Option<String>,
+    url: String,
+}
+
+struct FileFetch {
+    plan: FilePlan,
+}
+
+struct FileFetched {
+    plan: FilePlan,
+    result: Result<FetchedBody, JmapError>,
+}
+
+struct FetchedBody {
+    status: u16,
+    bytes: Vec<u8>,
+    etag: Option<String>,
+    content_type: Option<String>,
+    last_modified: Option<String>,
+}
+
+pub struct WebDavCtx<'a> {
+    pub client: &'a DavClient,
+    pub source_id: i64,
+    pub base_url: &'a str,
+    pub dav_connections: usize,
+    pub logger: Logger,
+}
+
+pub fn reconcile_filenodes(
+    conn: &mut Connection,
+    ctx: &WebDavCtx<'_>,
+    root: &DiscoveredCollection,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let client = ctx.client;
+    let source_id = ctx.source_id;
+    let base_url = ctx.base_url;
+    let dav_connections = ctx.dav_connections;
+    let logger = ctx.logger;
+    let absolute_root = absolute(base_url, root.href.as_str())?;
+    let root_local = super::collections::upsert_root_directory(conn, source_id, root)?;
+
+    let known: Vec<(String, i64)> =
+        dav_ids::collections_of_type(conn, source_id, dav_ids::FILE_NODE)
+            .map_err(|e| Error::Partial(e.to_string()))?
+            .into_iter()
+            .collect();
+    let mut seen: HashSet<String> = HashSet::new();
+    seen.insert(root.href.as_str().to_owned());
+
+    let mut visited: HashSet<String> = HashSet::new();
+    visited.insert(root.href.as_str().to_owned());
+    let mut queue: VecDeque<(String, String, i64)> = VecDeque::new();
+    queue.push_back((absolute_root, root.href.as_str().to_owned(), root_local));
+
+    let mut file_plans: Vec<FilePlan> = Vec::new();
+
+    while let Some((url, parent_href, parent_local)) = queue.pop_front() {
+        match walk_one(
+            conn,
+            client,
+            source_id,
+            WalkPos {
+                url: &url,
+                parent_href: &parent_href,
+                parent_local,
+            },
+            WalkState {
+                counts,
+                seen: &mut seen,
+                file_plans: &mut file_plans,
+            },
+            logger,
+        ) {
+            Ok(children) => {
+                for (child_url, child_href, child_local) in children {
+                    if visited.insert(child_href.clone()) {
+                        queue.push_back((child_url, child_href, child_local));
+                    } else {
+                        logger.warn(&format!("cycle detected at {child_href}; not recursing"));
+                    }
+                }
+            }
+            Err(e) => {
+                logger.warn(&format!("PROPFIND {url}: {e}"));
+                counts.failed += 1;
+            }
+        }
+    }
+
+    fetch_files_parallel(
+        conn,
+        client,
+        source_id,
+        dav_connections,
+        file_plans,
+        counts,
+        logger,
+    )?;
+
+    delete_vanished(conn, source_id, &known, &seen, counts, logger)?;
+
+    Ok(())
+}
+
+fn fetch_files_parallel(
+    conn: &mut Connection,
+    client: &DavClient,
+    source_id: i64,
+    dav_connections: usize,
+    plans: Vec<FilePlan>,
+    counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    if plans.is_empty() {
+        return Ok(());
+    }
+    let workers = dav_connections.clamp(1, 8);
+    let client_for_pool = client.clone();
+    let pool: Pool<FileFetch, FileFetched> = Pool::new(workers, move |job: FileFetch| {
+        let result = match client_for_pool.get_stream(&job.plan.url) {
+            Ok(mut s) => {
+                let mut bytes = Vec::new();
+                match s.read_to_end(&mut bytes) {
+                    Ok(_) => Ok(FetchedBody {
+                        status: s.status,
+                        bytes,
+                        etag: s.etag,
+                        content_type: s.content_type,
+                        last_modified: s.last_modified,
+                    }),
+                    Err(e) => Err(JmapError::Transport(format!("read body: {e}"))),
+                }
+            }
+            Err(e) => Err(e),
+        };
+        FileFetched {
+            plan: job.plan,
+            result,
+        }
+    });
+
+    let mut submitted: usize = 0;
+    for plan in plans {
+        pool.submit(FileFetch { plan });
+        submitted += 1;
+    }
+
+    const FILE_COMMIT_BATCH: usize = 16;
+    let mut tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let mut pending: usize = 0;
+    for _ in 0..submitted {
+        let fetched = match pool.results().recv() {
+            Ok(r) => r,
+            Err(_) => break,
+        };
+        match fetched.result {
+            Err(e) => {
+                logger.warn(&format!("file {}: {e}", fetched.plan.item_href));
+                counts.failed += 1;
+            }
+            Ok(body) if body.status == 404 || body.status == 410 => {}
+            Ok(body) if body.status >= 400 => {
+                logger.warn(&format!(
+                    "file {}: http {}",
+                    fetched.plan.item_href, body.status
+                ));
+                counts.failed += 1;
+            }
+            Ok(body) => match commit_file(&tx, source_id, &fetched.plan, body, logger) {
+                Ok(()) => {
+                    counts.fetched += 1;
+                    pending += 1;
+                    if pending >= FILE_COMMIT_BATCH {
+                        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                        tx = conn
+                            .unchecked_transaction()
+                            .map_err(|e| Error::Partial(e.to_string()))?;
+                        pending = 0;
+                    }
+                }
+                Err(e) => {
+                    logger.warn(&format!("file {}: {e}", fetched.plan.item_href));
+                    counts.failed += 1;
+                }
+            },
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    drop(pool);
+    Ok(())
+}
+
+fn commit_file(
+    tx: &rusqlite::Transaction<'_>,
+    source_id: i64,
+    plan: &FilePlan,
+    body: FetchedBody,
+    logger: Logger,
+) -> Result<(), Error> {
+    let etag = body
+        .etag
+        .clone()
+        .unwrap_or_else(|| plan.propfind_etag.clone());
+    let content_type = body
+        .content_type
+        .or_else(|| plan.propfind_content_type.clone());
+    let last_modified = normalise_dav_date(
+        &body
+            .last_modified
+            .or_else(|| plan.propfind_last_modified.clone()),
+    );
+    let created = format_or_now(&plan.propfind_creation_date)?;
+    let name = display_or_basename(plan, logger);
+
+    let blob_local =
+        db::blobs::intern_blob(tx, &body.bytes).map_err(|e| Error::Partial(e.to_string()))?;
+    if let Some(local) = plan.existing_local {
+        tx.execute(
+            "UPDATE file_nodes SET parent_id = ?1, node_type = 'file', blob_id = ?2,
+                                    target = NULL, name = ?3, media_type = ?4,
+                                    created = ?5, modified = ?6
+             WHERE id = ?7",
+            params![
+                plan.parent_local,
+                blob_local,
+                name,
+                content_type,
+                created,
+                last_modified,
+                local,
+            ],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        dav_ids::update_etag(tx, source_id, dav_ids::FILE_NODE, &plan.item_href, &etag)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    } else {
+        tx.execute(
+            "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type,
+                                      created, modified, is_subscribed, role)
+             VALUES (?1, 'file', ?2, NULL, ?3, ?4, ?5, ?6, 1, NULL)",
+            params![
+                plan.parent_local,
+                blob_local,
+                name,
+                content_type,
+                created,
+                last_modified,
+            ],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let local = tx.last_insert_rowid();
+        dav_ids::insert(
+            tx,
+            source_id,
+            dav_ids::FILE_NODE,
+            &plan.parent_href,
+            &plan.item_href,
+            &etag,
+            local,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn display_or_basename(plan: &FilePlan, logger: Logger) -> String {
+    let href = Href::from_normalised(plan.item_href.clone());
+    let basename = decoded_basename(&href);
+    if let Some(n) = &plan.propfind_displayname
+        && !n.trim().is_empty()
+        && n.trim() != basename
+        && logger.enabled(crate::logging::LEVEL_BODIES)
+    {
+        eprintln!(
+            "webdav: {} displayname={:?} differs from basename={basename:?}; using basename",
+            plan.item_href, n
+        );
+    }
+    basename
+}
+
+fn delete_vanished(
+    conn: &mut Connection,
+    source_id: i64,
+    known: &[(String, i64)],
+    seen: &HashSet<String>,
+    counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let mut vanished: Vec<(String, i64)> = known
+        .iter()
+        .filter(|(href, _)| !seen.contains(href))
+        .cloned()
+        .collect();
+    if vanished.is_empty() {
+        return Ok(());
+    }
+    vanished.sort_by(|a, b| {
+        let depth_a = a.0.matches('/').count();
+        let depth_b = b.0.matches('/').count();
+        depth_b.cmp(&depth_a).then_with(|| a.0.cmp(&b.0))
+    });
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    for (href, local_id) in &vanished {
+        if let Err(e) = tx.execute("DELETE FROM file_nodes WHERE id = ?1", params![local_id]) {
+            logger.warn(&format!(
+                "vanished file_node {href:?} delete failed: {e}; skipping"
+            ));
+            counts.failed += 1;
+            continue;
+        }
+        if let Err(e) = dav_ids::delete_item(&tx, source_id, dav_ids::FILE_NODE, href) {
+            logger.warn(&format!(
+                "vanished file_node {href:?} sync_id delete failed: {e}"
+            ));
+            counts.failed += 1;
+            continue;
+        }
+        counts.deleted += 1;
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!("webdav: deleted {} vanished file_nodes", vanished.len());
+    }
+    Ok(())
+}
+
+struct WalkPos<'a> {
+    url: &'a str,
+    parent_href: &'a str,
+    parent_local: i64,
+}
+
+struct WalkState<'a> {
+    counts: &'a mut TypeCounts,
+    seen: &'a mut HashSet<String>,
+    file_plans: &'a mut Vec<FilePlan>,
+}
+
+fn walk_one(
+    conn: &mut Connection,
+    client: &DavClient,
+    source_id: i64,
+    pos: WalkPos<'_>,
+    state: WalkState<'_>,
+    logger: Logger,
+) -> Result<Vec<(String, String, i64)>, Error> {
+    let url = pos.url;
+    let parent_href = pos.parent_href;
+    let parent_local = pos.parent_local;
+    let counts = state.counts;
+    let seen = state.seen;
+    let file_plans = state.file_plans;
+    let body = xml::propfind_webdav_listing();
+    let ms = client
+        .propfind_responses(url, 1, &body, url)
+        .map_err(Error::from)?;
+    if ms.status >= 400 {
+        return Err(Error::Partial(format!("http {}", ms.status)));
+    }
+    let parent_norm = crate::dav::href::normalise(url, "")
+        .map(|h| h.into_string())
+        .unwrap_or_default();
+    let mut children = Vec::new();
+    for r in ms.responses {
+        if r.href.as_str() == parent_norm || r.href.as_str() == parent_href {
+            continue;
+        }
+        seen.insert(r.href.as_str().to_owned());
+        if r.props.is_collection {
+            let local = upsert_directory(conn, source_id, &r, parent_local, counts)?;
+            let abs = absolute(url, r.href.as_str())?;
+            children.push((abs, r.href.as_str().to_owned(), local));
+        } else {
+            match plan_file(conn, source_id, &r, url, parent_local, parent_href) {
+                Ok(Some(plan)) => file_plans.push(plan),
+                Ok(None) => {}
+                Err(e) => {
+                    logger.warn(&format!("file {}: {e}", r.href.as_str()));
+                    counts.failed += 1;
+                }
+            }
+        }
+    }
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "dir {url}: queued {} files (fetched-so-far={} failed-so-far={})",
+            file_plans.len(),
+            counts.fetched,
+            counts.failed
+        );
+    }
+    Ok(children)
+}
+
+fn plan_file(
+    conn: &Connection,
+    source_id: i64,
+    response: &DavResponse,
+    collection_url: &str,
+    parent_local: i64,
+    parent_href: &str,
+) -> Result<Option<FilePlan>, Error> {
+    let item_href = response.href.as_str().to_owned();
+    let propfind_etag = response.props.etag.clone().unwrap_or_default();
+    let existing = dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &item_href)
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    if let Some(local) = existing {
+        let stored_etag: Option<String> = conn
+            .query_row(
+                "SELECT etag FROM sync_id_dav
+                 WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3",
+                params![source_id, dav_ids::FILE_NODE, item_href],
+                |r| r.get(0),
+            )
+            .optional()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        if let Some(stored) = stored_etag.as_deref()
+            && !propfind_etag.is_empty()
+            && stored == propfind_etag
+        {
+            return Ok(None);
+        }
+        let url = absolute(collection_url, &item_href)?;
+        Ok(Some(FilePlan {
+            item_href,
+            parent_href: parent_href.to_owned(),
+            parent_local,
+            existing_local: Some(local),
+            propfind_etag,
+            propfind_content_type: response.props.content_type.clone(),
+            propfind_last_modified: response.props.last_modified.clone(),
+            propfind_creation_date: response.props.creation_date.clone(),
+            propfind_displayname: response.props.displayname.clone(),
+            url,
+        }))
+    } else {
+        let url = absolute(collection_url, &item_href)?;
+        Ok(Some(FilePlan {
+            item_href,
+            parent_href: parent_href.to_owned(),
+            parent_local,
+            existing_local: None,
+            propfind_etag,
+            propfind_content_type: response.props.content_type.clone(),
+            propfind_last_modified: response.props.last_modified.clone(),
+            propfind_creation_date: response.props.creation_date.clone(),
+            propfind_displayname: response.props.displayname.clone(),
+            url,
+        }))
+    }
+}
+
+fn upsert_directory(
+    conn: &mut Connection,
+    source_id: i64,
+    response: &DavResponse,
+    parent_local: i64,
+    counts: &mut TypeCounts,
+) -> Result<i64, Error> {
+    let item_href = response.href.as_str().to_owned();
+    let name = display_or_path(&response.props.displayname, &response.href);
+    let modified = normalise_dav_date(&response.props.last_modified);
+    let existing = dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &item_href)
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    if let Some(local) = existing {
+        tx.execute(
+            "UPDATE file_nodes SET parent_id = ?1, name = ?2, modified = ?3
+             WHERE id = ?4",
+            params![parent_local, name, modified, local],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+        counts.fetched += 1;
+        Ok(local)
+    } else {
+        let created = format_or_now(&response.props.creation_date)?;
+        tx.execute(
+            "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type,
+                                      created, modified, is_subscribed, role)
+             VALUES (?1, 'directory', NULL, NULL, ?2, NULL, ?3, ?4, 1, NULL)",
+            params![parent_local, name, created, modified],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let local = tx.last_insert_rowid();
+        let parent_href = parent_href_string(&response.href);
+        dav_ids::insert(
+            &tx,
+            source_id,
+            dav_ids::FILE_NODE,
+            &parent_href,
+            &item_href,
+            "",
+            local,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+        counts.created += 1;
+        Ok(local)
+    }
+}
+
+pub(super) fn parent_href_string(href: &Href) -> String {
+    let s = href.as_str();
+    let trimmed = s.trim_end_matches('/');
+    if let Some(slash) = trimmed.rfind('/') {
+        s[..=slash].to_owned()
+    } else {
+        "/".to_owned()
+    }
+}
+
+fn absolute(base: &str, href: &str) -> Result<String, Error> {
+    join_absolute(base, href).map_err(|e| Error::Partial(e.to_string()))
+}
+
+fn display_or_path(name: &Option<String>, href: &Href) -> String {
+    if let Some(n) = name
+        && !n.trim().is_empty()
+    {
+        return n.clone();
+    }
+    decoded_basename(href)
+}
+
+fn decoded_basename(href: &Href) -> String {
+    let c = last_path_component(href);
+    if c.is_empty() { "_".to_owned() } else { c }
+}
+
+fn normalise_dav_date(input: &Option<String>) -> Option<String> {
+    let raw = input.as_deref()?.trim();
+    if raw.is_empty() {
+        return None;
+    }
+    if let Ok(t) = time::OffsetDateTime::parse(raw, &Rfc3339) {
+        return t.format(&Rfc3339).ok();
+    }
+    let imf_fixdate = time::macros::format_description!(
+        "[weekday repr:short], [day] [month repr:short] [year] \
+         [hour]:[minute]:[second] GMT"
+    );
+    if let Ok(p) = time::PrimitiveDateTime::parse(raw, imf_fixdate) {
+        return p.assume_utc().format(&Rfc3339).ok();
+    }
+    None
+}
+
+fn format_or_now(input: &Option<String>) -> Result<String, Error> {
+    if let Some(s) = normalise_dav_date(input) {
+        return Ok(s);
+    }
+    time::OffsetDateTime::now_utc()
+        .format(&Rfc3339)
+        .map_err(|e| Error::Partial(format!("clock: {e}")))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parent_href_string_of_root_is_root() {
+        let h = Href::from_normalised("/".to_owned());
+        assert_eq!(parent_href_string(&h), "/");
+    }
+
+    #[test]
+    fn parent_href_string_of_dir_strips_trailing_segment() {
+        let h = Href::from_normalised("/dav/file/u/work/".to_owned());
+        assert_eq!(parent_href_string(&h), "/dav/file/u/");
+    }
+
+    #[test]
+    fn decoded_basename_handles_percent_encoding() {
+        let h = Href::from_normalised("/dav/file/u/work/note%201.txt".to_owned());
+        assert_eq!(decoded_basename(&h), "note 1.txt");
+    }
+
+    fn plan_with(displayname: Option<&str>, href: &str) -> FilePlan {
+        FilePlan {
+            item_href: href.to_owned(),
+            parent_href: "/dav/file/u/".to_owned(),
+            parent_local: 1,
+            existing_local: None,
+            propfind_etag: String::new(),
+            propfind_content_type: None,
+            propfind_last_modified: None,
+            propfind_creation_date: None,
+            propfind_displayname: displayname.map(str::to_owned),
+            url: format!("https://x{href}"),
+        }
+    }
+
+    #[test]
+    fn display_or_basename_prefers_url_basename_over_displayname() {
+        let plan = plan_with(Some("Pretty Name.txt"), "/dav/file/u/photo.jpg");
+        let logger = Logger::from_flags(false, 0);
+        assert_eq!(display_or_basename(&plan, logger), "photo.jpg");
+    }
+
+    #[test]
+    fn display_or_basename_returns_basename_when_no_displayname() {
+        let plan = plan_with(None, "/dav/file/u/photo.jpg");
+        let logger = Logger::from_flags(false, 0);
+        assert_eq!(display_or_basename(&plan, logger), "photo.jpg");
+    }
+
+    #[test]
+    fn normalise_dav_date_parses_imf_fixdate() {
+        let n = normalise_dav_date(&Some("Wed, 27 May 2026 07:59:07 GMT".to_owned()))
+            .expect("imf parse");
+        assert!(n.starts_with("2026-05-27T07:59:07"));
+    }
+
+    #[test]
+    fn normalise_dav_date_returns_none_on_garbage() {
+        assert!(normalise_dav_date(&Some("not a date".to_owned())).is_none());
+        assert!(normalise_dav_date(&None).is_none());
+    }
+
+    #[test]
+    fn visited_hashset_breaks_recursion_on_cyclic_listing() {
+        let mut visited: HashSet<String> = HashSet::new();
+        visited.insert("/files/u/".to_owned());
+        assert!(visited.insert("/files/u/a/".to_owned()));
+        assert!(visited.insert("/files/u/a/b/".to_owned()));
+        assert!(
+            !visited.insert("/files/u/".to_owned()),
+            "cycle back to root not re-recursed"
+        );
+        assert!(
+            !visited.insert("/files/u/a/".to_owned()),
+            "cycle back to a/ not re-recursed"
+        );
+    }
+}

src/sync/import_exchange_ews.rs

@@ -0,0 +1,15 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod attachments;
+pub mod calendar;
+pub mod contacts;
+pub mod coordinator;
+pub mod folders;
+pub mod items;
+pub mod messages;
+
+pub use coordinator::{EwsAuth, EwsImportConfig, run};

src/sync/import_exchange_ews/attachments.rs

@@ -0,0 +1,103 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD;
+
+use crate::db::blobs;
+use crate::error::Error;
+use crate::exchange_ews::error::EwsError;
+use crate::exchange_ews::parse::parse_get_attachment_inline;
+use crate::exchange_ews::xml::get_attachment_body;
+
+use super::items::ItemRunCtx;
+
+#[derive(Debug, Clone)]
+pub struct FetchedAttachment {
+    pub attachment_id: String,
+    pub bytes: Vec<u8>,
+    pub media_type: String,
+}
+
+pub fn fetch_contact_photo(
+    conn: &rusqlite::Connection,
+    ctx: &ItemRunCtx<'_>,
+    attachment_id: &str,
+) -> Result<Option<(i64, String)>, Error> {
+    let Some(att) = fetch_attachment_bytes(ctx, attachment_id).map_err(Error::from)? else {
+        return Ok(None);
+    };
+    let blob_id = intern_attachment(conn, &att.bytes)?;
+    Ok(Some((blob_id, att.media_type)))
+}
+
+pub fn fetch_attachment_bytes(
+    ctx: &ItemRunCtx<'_>,
+    attachment_id: &str,
+) -> Result<Option<FetchedAttachment>, EwsError> {
+    let mut out = fetch_attachments(ctx, &[attachment_id])?;
+    Ok(out.pop())
+}
+
+pub fn fetch_attachments(
+    ctx: &ItemRunCtx<'_>,
+    attachment_ids: &[&str],
+) -> Result<Vec<FetchedAttachment>, EwsError> {
+    if attachment_ids.is_empty() {
+        return Ok(Vec::new());
+    }
+    let batch = ctx.attachment_batch.max(1);
+    let mut out: Vec<FetchedAttachment> = Vec::with_capacity(attachment_ids.len());
+    for chunk in attachment_ids.chunks(batch) {
+        let body = get_attachment_body(chunk);
+        let resp = ctx.client.call(ctx.url, "GetAttachment", &body)?;
+        let inline = parse_get_attachment_inline(&resp.body)?;
+        for att in inline {
+            let cleaned = strip_ascii_whitespace(att.content_base64.as_bytes());
+            if cleaned.is_empty() {
+                continue;
+            }
+            let bytes = STANDARD.decode(&cleaned).map_err(|e| {
+                EwsError::Malformed(format!("attachment {}: base64: {e}", att.attachment_id))
+            })?;
+            let media_type = att
+                .content_type
+                .unwrap_or_else(|| "application/octet-stream".to_owned());
+            out.push(FetchedAttachment {
+                attachment_id: att.attachment_id,
+                bytes,
+                media_type,
+            });
+        }
+    }
+    Ok(out)
+}
+
+pub fn intern_attachment(conn: &rusqlite::Connection, bytes: &[u8]) -> Result<i64, Error> {
+    blobs::intern_blob(conn, bytes).map_err(|e| Error::Partial(e.to_string()))
+}
+
+fn strip_ascii_whitespace(input: &[u8]) -> Vec<u8> {
+    let mut out = Vec::with_capacity(input.len());
+    for b in input {
+        if !matches!(b, b' ' | b'\t' | b'\n' | b'\r') {
+            out.push(*b);
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::strip_ascii_whitespace;
+
+    #[test]
+    fn strips_whitespace_efficiently() {
+        let input = b"AB CD\nEF\tGH\r\nIJ";
+        let cleaned = strip_ascii_whitespace(input);
+        assert_eq!(cleaned, b"ABCDEFGHIJ");
+    }
+}

src/sync/import_exchange_ews/calendar.rs

@@ -0,0 +1,292 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::{Connection, params};
+use serde_json::Value;
+
+use crate::db::exchange_ews_ids;
+use crate::error::Error;
+use crate::exchange_ews::calendar_map::to_jscalendar;
+use crate::exchange_ews::parse::{CalendarItemRaw, parse_calendar_item};
+use crate::exchange_ews::types::{CalendarItemType, ItemId};
+use crate::exchange_ews::xml::ItemShape;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+
+use super::attachments::{fetch_attachments, intern_attachment};
+use super::folders::FolderPlan;
+use super::items::{
+    EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
+};
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    plan: &FolderPlan,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for folder in &plan.calendar {
+        let folder_id = &folder.folder.folder_id;
+        let local_folder_id = match exchange_ews_ids::local_for_item(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            &folder_id.id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        {
+            Some(id) => id,
+            None => continue,
+        };
+        if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) {
+            ctx.logger
+                .warn(&format!("calendar folder {} failed: {}", folder_id.id, e));
+            counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn reconcile_one(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    folder: &crate::exchange_ews::types::FolderId,
+    local_folder_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let prior = exchange_ews_ids::get_sync_state(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CALENDAR,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let mut outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?;
+    outcome.items.retain(|s| {
+        matches!(
+            s.element.to_ascii_lowercase().as_str(),
+            "calendaritem" | "item"
+        )
+    });
+    let local = exchange_ews_ids::items_in_folder(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CALENDAR_EVENT,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let plan = plan_for(&outcome, &local);
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS calendar folder {}: new={} changed={} vanished={}",
+            folder.id,
+            plan.new.len(),
+            plan.present_changed.len(),
+            plan.vanished.len()
+        );
+    }
+    let mut to_fetch: Vec<ItemId> = plan.new.clone();
+    for (id, _) in &plan.present_changed {
+        to_fetch.push(id.clone());
+    }
+    if !to_fetch.is_empty() {
+        let outcome = get_items(ctx, ItemShape::CalendarItem, &to_fetch).map_err(Error::from)?;
+        counts.failed += outcome.failed_items;
+        for msg in outcome.messages {
+            if !msg.success {
+                if matches!(
+                    msg.response_code,
+                    crate::exchange_ews::types::ResponseCode::ItemNotFound
+                ) {
+                    counts.skipped += 1;
+                } else {
+                    counts.failed += 1;
+                    ctx.logger
+                        .warn(&format!("GetItem (calendar) error: {}", msg.response_code));
+                }
+                continue;
+            }
+            let parsed = parse_calendar_item(&msg.inner_xml).map_err(Error::from)?;
+            if parsed.id.id.is_empty() {
+                counts.failed += 1;
+                continue;
+            }
+            if matches!(
+                parsed.calendar_item_type,
+                Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception)
+            ) {
+                counts.skipped += 1;
+                continue;
+            }
+            let existing = plan
+                .present_changed
+                .iter()
+                .find(|(id, _)| id.id == parsed.id.id)
+                .map(|(_, local)| *local);
+            apply_event(
+                conn,
+                ctx,
+                &parsed,
+                local_folder_id,
+                &folder.id,
+                existing,
+                counts,
+            )?;
+        }
+    }
+    delete_vanished(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CALENDAR_EVENT,
+        "calendar_events",
+        &plan.vanished,
+        counts,
+    )?;
+    if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode {
+        exchange_ews_ids::set_sync_state(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            &folder.id,
+            new_sync_state,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn apply_event(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    raw: &CalendarItemRaw,
+    local_folder_id: i64,
+    folder_id: &str,
+    existing_local_id: Option<i64>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let mut built = to_jscalendar(raw);
+    attach_calendar_links(conn, ctx, raw, &mut built.data)?;
+    let calendar_ids = serde_json::json!([local_folder_id]).to_string();
+    let data = built.data.to_string();
+    let is_draft = if built.is_draft { 1 } else { 0 };
+    let use_default_alerts = if built.use_default_alerts { 1 } else { 0 };
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    match existing_local_id {
+        Some(id) => {
+            tx.execute(
+                "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, \
+                 use_default_alerts = ?3, data = ?4, data_type = 'Event' WHERE id = ?5",
+                params![calendar_ids, is_draft, use_default_alerts, data, id],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            exchange_ews_ids::update_change_key(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CALENDAR_EVENT,
+                &raw.id.id,
+                &raw.id.change_key,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.fetched += 1;
+        }
+        None => {
+            tx.execute(
+                "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) \
+                 VALUES (?1, ?2, ?3, ?4, 'Event')",
+                params![calendar_ids, is_draft, use_default_alerts, data],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            let new_id = tx.last_insert_rowid();
+            exchange_ews_ids::insert(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CALENDAR_EVENT,
+                folder_id,
+                &raw.id.id,
+                &raw.id.change_key,
+                new_id,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.created += 1;
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}
+
+fn attach_calendar_links(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    raw: &CalendarItemRaw,
+    data: &mut Value,
+) -> Result<(), Error> {
+    let refs: Vec<_> = raw
+        .attachments
+        .iter()
+        .filter(|a| !a.is_item_attachment && !a.attachment_id.is_empty())
+        .collect();
+    if refs.is_empty() {
+        return Ok(());
+    }
+    let ids: Vec<&str> = refs.iter().map(|a| a.attachment_id.as_str()).collect();
+    let fetched = match fetch_attachments(ctx, &ids) {
+        Ok(f) => f,
+        Err(e) => {
+            ctx.logger.warn(&format!(
+                "calendar attachments fetch failed: {e}; continuing without enclosures"
+            ));
+            return Ok(());
+        }
+    };
+    let mut links = serde_json::Map::new();
+    for (idx, att) in (1u32..).zip(fetched) {
+        let blob_id = intern_attachment(conn, &att.bytes)?;
+        let declared = refs
+            .iter()
+            .find(|r| r.attachment_id == att.attachment_id)
+            .and_then(|r| r.content_type.as_deref());
+        let name = refs
+            .iter()
+            .find(|r| r.attachment_id == att.attachment_id)
+            .and_then(|r| r.name.clone());
+        let key = idx.to_string();
+        let mut entry = serde_json::Map::new();
+        entry.insert("@type".to_owned(), Value::String("Link".to_owned()));
+        entry.insert("@blob".to_owned(), Value::from(blob_id));
+        entry.insert(
+            "contentType".to_owned(),
+            Value::String(content_type_or_from(declared, &att.media_type)),
+        );
+        if let Some(n) = name {
+            entry.insert("title".to_owned(), Value::String(n));
+        }
+        entry.insert("rel".to_owned(), Value::String("enclosure".to_owned()));
+        links.insert(key, Value::Object(entry));
+    }
+    if links.is_empty() {
+        return Ok(());
+    }
+    if let Some(obj) = data.as_object_mut() {
+        if let Some(existing) = obj.get_mut("links").and_then(Value::as_object_mut) {
+            for (k, v) in links {
+                existing.insert(k, v);
+            }
+        } else {
+            obj.insert("links".to_owned(), Value::Object(links));
+        }
+    }
+    Ok(())
+}
+
+fn content_type_or_from(declared: Option<&str>, fallback: &str) -> String {
+    declared
+        .filter(|s| !s.is_empty())
+        .map(str::to_owned)
+        .unwrap_or_else(|| fallback.to_owned())
+}

src/sync/import_exchange_ews/contacts.rs

@@ -0,0 +1,227 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::{Connection, params};
+
+use crate::db::exchange_ews_ids;
+use crate::error::Error;
+use crate::exchange_ews::contact_map::{synthetic_uid, to_jscontact};
+use crate::exchange_ews::parse::{ContactItemRaw, parse_contact_item};
+use crate::exchange_ews::types::ItemId;
+use crate::exchange_ews::xml::ItemShape;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+
+use super::attachments::fetch_contact_photo;
+use super::folders::FolderPlan;
+use super::items::{
+    EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
+};
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    plan: &FolderPlan,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for folder in &plan.contacts {
+        let folder_id = &folder.folder.folder_id;
+        let local_folder_id = match exchange_ews_ids::local_for_item(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            &folder_id.id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        {
+            Some(id) => id,
+            None => continue,
+        };
+        if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) {
+            ctx.logger
+                .warn(&format!("contact folder {} failed: {}", folder_id.id, e));
+            counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn reconcile_one(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    folder: &crate::exchange_ews::types::FolderId,
+    local_folder_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let prior = exchange_ews_ids::get_sync_state(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::ADDRESS_BOOK,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?;
+    let local = exchange_ews_ids::items_in_folder(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CONTACT_CARD,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let plan = plan_for(&outcome, &local);
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS contacts folder {}: new={} changed={} vanished={}",
+            folder.id,
+            plan.new.len(),
+            plan.present_changed.len(),
+            plan.vanished.len()
+        );
+    }
+    let mut to_fetch: Vec<ItemId> = plan.new.clone();
+    for (id, _) in &plan.present_changed {
+        to_fetch.push(id.clone());
+    }
+    if !to_fetch.is_empty() {
+        let outcome = get_items(ctx, ItemShape::Contact, &to_fetch).map_err(Error::from)?;
+        counts.failed += outcome.failed_items;
+        for msg in outcome.messages {
+            if !msg.success {
+                if matches!(
+                    msg.response_code,
+                    crate::exchange_ews::types::ResponseCode::ItemNotFound
+                ) {
+                    counts.skipped += 1;
+                } else {
+                    counts.failed += 1;
+                    ctx.logger
+                        .warn(&format!("GetItem (contact) error: {}", msg.response_code));
+                }
+                continue;
+            }
+            let parsed = parse_contact_item(&msg.inner_xml).map_err(Error::from)?;
+            if parsed.id.id.is_empty() {
+                counts.failed += 1;
+                continue;
+            }
+            let existing = plan
+                .present_changed
+                .iter()
+                .find(|(id, _)| id.id == parsed.id.id)
+                .map(|(_, local)| *local);
+            apply_contact(
+                conn,
+                ctx,
+                &parsed,
+                local_folder_id,
+                &folder.id,
+                existing,
+                counts,
+            )?;
+        }
+    }
+    delete_vanished(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CONTACT_CARD,
+        "contact_cards",
+        &plan.vanished,
+        counts,
+    )?;
+    if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode {
+        exchange_ews_ids::set_sync_state(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            &folder.id,
+            new_sync_state,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn apply_contact(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    item: &ContactItemRaw,
+    local_folder_id: i64,
+    folder_id: &str,
+    existing_local_id: Option<i64>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let mut card = to_jscontact(item);
+    if let Some(photo_att) = item.attachments.iter().find(|a| a.is_contact_photo) {
+        match fetch_contact_photo(conn, ctx, &photo_att.attachment_id) {
+            Ok(Some((blob_id, media_type))) => {
+                let mut media = serde_json::Map::new();
+                media.insert(
+                    "photo".to_owned(),
+                    serde_json::json!({
+                        "@type": "Media",
+                        "kind": "photo",
+                        "@blob": blob_id,
+                        "mediaType": media_type,
+                    }),
+                );
+                if let Some(m) = card.as_object_mut() {
+                    m.insert("media".to_owned(), serde_json::Value::Object(media));
+                }
+            }
+            Ok(None) => {}
+            Err(e) => {
+                ctx.logger
+                    .warn(&format!("contact photo fetch failed: {e}; continuing"));
+            }
+        }
+    }
+    let uid = synthetic_uid(&item.id.id);
+    let address_book_ids = serde_json::json!([local_folder_id]).to_string();
+    let data = card.to_string();
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    match existing_local_id {
+        Some(id) => {
+            tx.execute(
+                "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 WHERE id = ?4",
+                params![uid, address_book_ids, data, id],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            exchange_ews_ids::update_change_key(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CONTACT_CARD,
+                &item.id.id,
+                &item.id.change_key,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.fetched += 1;
+        }
+        None => {
+            tx.execute(
+                "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)",
+                params![uid, address_book_ids, data],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            let new_id = tx.last_insert_rowid();
+            exchange_ews_ids::insert(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CONTACT_CARD,
+                folder_id,
+                &item.id.id,
+                &item.id.change_key,
+                new_id,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.created += 1;
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}

src/sync/import_exchange_ews/coordinator.rs

@@ -0,0 +1,521 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::Connection;
+
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::error::Error;
+use crate::exchange_ews::EwsClient;
+use crate::exchange_ews::autodiscover::{DiscoveryResult, discover};
+use crate::exchange_ews::oauth::{AcquiredToken, OAuthFlow, acquire};
+use crate::exchange_ews::types::MailboxKind;
+use crate::jmap::http::{Auth, RetryPolicy};
+use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS};
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+
+use super::folders::{self, plan_folders};
+use super::{calendar, contacts, messages};
+
+#[derive(Debug, Clone)]
+pub enum EwsAuth {
+    Basic { user: String, password: String },
+    Bearer { token: String },
+    OAuth(OAuthFlow),
+}
+
+#[derive(Debug, Clone)]
+pub struct EwsImportConfig {
+    pub url: Option<String>,
+    pub mailbox: Option<String>,
+    pub mailbox_kind: MailboxKind,
+    pub auth: EwsAuth,
+    pub ews_connections: usize,
+    pub getitem_batch: usize,
+    pub attachment_batch: usize,
+    pub use_syncfolderitems: bool,
+    pub allow_source_change: bool,
+}
+
+pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    let mut conn = db::init::open(&common.archive)?;
+
+    let (auth, acquired) = resolve_auth(&config.auth, common.allow_invalid_certs)?;
+    let discovery = run_autodiscover(&config, &acquired, common.allow_invalid_certs)?;
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS discovery: url={} source={:?}",
+            discovery.ews_url, discovery.source
+        );
+    }
+    let mailbox = resolve_mailbox(&config, &acquired)?;
+    let account_id = synthetic_account_id(&mailbox, config.mailbox_kind);
+    let session_url = normalise_ews_url(&discovery.ews_url);
+    enforce_basic_auth_policy(&config.auth, &session_url)?;
+    let key = SourceKey {
+        kind: "exchange_ews".to_owned(),
+        session_url: session_url.clone(),
+        account_id: account_id.clone(),
+    };
+
+    if !common.dry_run
+        && let Some((url, acc)) =
+            db::sources::conflicting_source(&conn, "exchange_ews", &session_url, &account_id)?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records exchange_ews source ({url}, account {acc}); \
+             pass --allow-source-change to import a different account"
+        )));
+    }
+
+    let client = EwsClient::new(
+        auth,
+        RetryPolicy::new(common.max_retries),
+        common.allow_invalid_certs,
+    );
+    client.set_logger(logger);
+    if matches!(config.mailbox_kind, MailboxKind::PublicFolders) {
+        client.set_anchor_mailbox(None);
+    } else {
+        client.set_anchor_mailbox(Some(mailbox.clone()));
+    }
+    if let EwsAuth::OAuth(OAuthFlow::ClientCredentials { .. }) = &config.auth {
+        client.set_impersonation(Some(mailbox.clone()));
+    }
+    spawn_token_refresher(
+        &client,
+        &config.auth,
+        &acquired,
+        common.allow_invalid_certs,
+        logger,
+    );
+
+    let username = match &config.auth {
+        EwsAuth::Basic { user, .. } => user.clone(),
+        EwsAuth::Bearer { .. } | EwsAuth::OAuth(_) => acquired
+            .as_ref()
+            .and_then(|a| a.upn.clone())
+            .unwrap_or_else(|| mailbox.clone()),
+    };
+
+    if common.dry_run {
+        let summary = run_dry(&conn, &client, &session_url, &config, logger)?;
+        return Ok(summary);
+    }
+
+    let account_name = acquired.as_ref().and_then(|a| a.name.clone());
+    let source_id = db::sources::upsert_source(&conn, &key, account_name.as_deref(), &username)?;
+
+    let mut summary = Summary::default();
+    let mut mailbox_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut addressbook_counts = TypeCounts::default();
+    let mut email_counts = TypeCounts::default();
+    let mut calendar_event_counts = TypeCounts::default();
+    let mut contact_counts = TypeCounts::default();
+
+    let plan =
+        plan_folders(&client, &session_url, config.mailbox_kind, logger).map_err(Error::from)?;
+
+    folders::reconcile(
+        &mut conn,
+        source_id,
+        &plan,
+        &mut mailbox_counts,
+        &mut calendar_counts,
+        &mut addressbook_counts,
+        logger,
+    )?;
+
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "import: mailbox={} calendar={} addressbook={}",
+            mailbox_counts.created + mailbox_counts.fetched,
+            calendar_counts.created + calendar_counts.fetched,
+            addressbook_counts.created + addressbook_counts.fetched,
+        );
+    }
+
+    let item_ctx = super::items::ItemRunCtx {
+        client: &client,
+        url: &session_url,
+        source_id,
+        batch_size: config.getitem_batch.max(1),
+        attachment_batch: config.attachment_batch.max(1),
+        connections: config.ews_connections.clamp(1, 8),
+        use_syncfolderitems: config.use_syncfolderitems,
+        logger,
+    };
+
+    messages::reconcile_all(&mut conn, &item_ctx, &plan, &mut email_counts)?;
+    contacts::reconcile_all(&mut conn, &item_ctx, &plan, &mut contact_counts)?;
+    calendar::reconcile_all(&mut conn, &item_ctx, &plan, &mut calendar_event_counts)?;
+
+    summary.per_type.push(("mailbox", mailbox_counts));
+    summary.per_type.push(("calendar", calendar_counts));
+    summary.per_type.push(("addressbook", addressbook_counts));
+    summary.per_type.push(("email", email_counts));
+    summary
+        .per_type
+        .push(("calendarevent", calendar_event_counts));
+    summary.per_type.push(("contactcard", contact_counts));
+
+    if !summary.any_failed()
+        && let Err(e) = run_gc(&conn)
+    {
+        logger.warn(&format!("blob GC skipped: {e}"));
+    }
+
+    summary.retries_observed = client.retries_observed();
+    summary.retry_after_sleeps = client.retry_after_sleeps();
+    Ok(summary)
+}
+
+fn run_dry(
+    _conn: &Connection,
+    client: &EwsClient,
+    session_url: &str,
+    config: &EwsImportConfig,
+    logger: crate::logging::Logger,
+) -> Result<Summary, Error> {
+    let plan =
+        plan_folders(client, session_url, config.mailbox_kind, logger).map_err(Error::from)?;
+    let mut summary = Summary::default();
+    let mut mailbox_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut addressbook_counts = TypeCounts::default();
+    mailbox_counts.created = plan.mail.len() as u64;
+    calendar_counts.created = plan.calendar.len() as u64;
+    addressbook_counts.created = plan.contacts.len() as u64;
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "dry-run: mailbox={} calendar={} addressbook={}",
+            mailbox_counts.created, calendar_counts.created, addressbook_counts.created
+        );
+    }
+    summary.per_type.push(("mailbox", mailbox_counts));
+    summary.per_type.push(("calendar", calendar_counts));
+    summary.per_type.push(("addressbook", addressbook_counts));
+    Ok(summary)
+}
+
+fn resolve_auth(
+    auth: &EwsAuth,
+    allow_invalid_certs: bool,
+) -> Result<(Auth, Option<AcquiredToken>), Error> {
+    match auth {
+        EwsAuth::Basic { user, password } => Ok((
+            Auth::Basic {
+                user: user.clone(),
+                password: password.clone(),
+            },
+            None,
+        )),
+        EwsAuth::Bearer { token } => {
+            let acq = acquire(
+                &OAuthFlow::PreAcquired {
+                    token: token.clone(),
+                },
+                allow_invalid_certs,
+            )
+            .map_err(Error::from)?;
+            Ok((
+                Auth::Bearer {
+                    token: acq.access_token.clone(),
+                },
+                Some(acq),
+            ))
+        }
+        EwsAuth::OAuth(flow) => {
+            let acq = acquire(flow, allow_invalid_certs).map_err(Error::from)?;
+            Ok((
+                Auth::Bearer {
+                    token: acq.access_token.clone(),
+                },
+                Some(acq),
+            ))
+        }
+    }
+}
+
+fn run_autodiscover(
+    config: &EwsImportConfig,
+    acquired: &Option<AcquiredToken>,
+    allow_invalid_certs: bool,
+) -> Result<DiscoveryResult, Error> {
+    let email = config
+        .mailbox
+        .clone()
+        .or_else(|| acquired.as_ref().and_then(|a| a.upn.clone()));
+    let result = discover(
+        config.url.as_deref(),
+        email.as_deref(),
+        None,
+        allow_invalid_certs,
+    )
+    .map_err(Error::from)?;
+    Ok(result)
+}
+
+fn resolve_mailbox(
+    config: &EwsImportConfig,
+    acquired: &Option<AcquiredToken>,
+) -> Result<String, Error> {
+    if let Some(mb) = config.mailbox.as_ref() {
+        return Ok(mb.clone());
+    }
+    if let Some(acq) = acquired.as_ref()
+        && let Some(upn) = acq.upn.as_ref()
+    {
+        return Ok(upn.clone());
+    }
+    if let EwsAuth::Basic { user, .. } = &config.auth
+        && user.contains('@')
+    {
+        return Ok(user.clone());
+    }
+    Err(Error::Account(
+        "could not resolve mailbox SMTP; pass --mailbox".to_owned(),
+    ))
+}
+
+pub fn synthetic_account_id(mailbox: &str, kind: MailboxKind) -> String {
+    match kind {
+        MailboxKind::Primary => mailbox.to_owned(),
+        MailboxKind::Archive => format!("{mailbox}#archive"),
+        MailboxKind::PublicFolders => {
+            let domain = mailbox.split('@').nth(1).unwrap_or("");
+            format!("__public_folders__@{domain}")
+        }
+    }
+}
+
+pub fn normalise_ews_url(raw: &str) -> String {
+    let Ok(parsed) = url::Url::parse(raw) else {
+        return raw.to_owned();
+    };
+    let scheme = parsed.scheme().to_ascii_lowercase();
+    let host = parsed.host_str().unwrap_or("").to_ascii_lowercase();
+    let default_port = match scheme.as_str() {
+        "https" => 443,
+        "http" => 80,
+        _ => 0,
+    };
+    let port = parsed.port().filter(|p| *p != default_port);
+    let path = if parsed.path().is_empty() || parsed.path() == "/" {
+        "/EWS/Exchange.asmx".to_owned()
+    } else {
+        let last = parsed.path().rsplit('/').next().unwrap_or("");
+        if last.eq_ignore_ascii_case("Exchange.asmx") {
+            let mut p = String::new();
+            let mut parts: Vec<&str> = parsed.path().split('/').collect();
+            if let Some(last_part) = parts.last_mut() {
+                *last_part = "Exchange.asmx";
+            }
+            if let Some(second_last_idx) = parts.len().checked_sub(2)
+                && parts[second_last_idx].eq_ignore_ascii_case("EWS")
+            {
+                parts[second_last_idx] = "EWS";
+            }
+            p.push_str(&parts.join("/"));
+            p
+        } else {
+            parsed.path().to_owned()
+        }
+    };
+    match port {
+        Some(p) => format!("{scheme}://{host}:{p}{path}"),
+        None => format!("{scheme}://{host}{path}"),
+    }
+}
+
+fn enforce_basic_auth_policy(auth: &EwsAuth, url: &str) -> Result<(), Error> {
+    if let EwsAuth::Basic { .. } = auth {
+        let parsed = url::Url::parse(url)
+            .map_err(|e| Error::Usage(format!("invalid EWS URL {url:?}: {e}")))?;
+        if parsed.scheme() != "https" {
+            return Err(Error::Connection(
+                "EWS requires https:// (basic auth refused on cleartext)".to_owned(),
+            ));
+        }
+        let host = parsed.host_str().unwrap_or("").to_ascii_lowercase();
+        let blocked = [
+            "outlook.office365.com",
+            "outlook.office.com",
+            "outlook.office365.us",
+            "office365.us",
+        ];
+        if blocked.iter().any(|b| host == *b || host.ends_with(b)) {
+            return Err(Error::Connection(
+                "Basic auth is disabled on Exchange Online; use --auth-bearer".to_owned(),
+            ));
+        }
+    }
+    Ok(())
+}
+
+fn spawn_token_refresher(
+    client: &EwsClient,
+    auth: &EwsAuth,
+    initial: &Option<AcquiredToken>,
+    allow_invalid_certs: bool,
+    logger: crate::logging::Logger,
+) {
+    let flow = match auth {
+        EwsAuth::OAuth(flow) => flow.clone(),
+        _ => return,
+    };
+    let exp = initial.as_ref().and_then(|t| {
+        crate::exchange_ews::oauth::decode_jwt_claims(&t.access_token).and_then(|c| c.exp)
+    });
+    let Some(exp_secs) = exp else {
+        return;
+    };
+    let client = client.clone();
+    let initial_refresh = initial.as_ref().and_then(|t| t.refresh_token.clone());
+    std::thread::Builder::new()
+        .name("vandelay-ews-token-refresh".to_owned())
+        .spawn(move || {
+            let mut next_exp = exp_secs;
+            let mut refresh_token = initial_refresh;
+            loop {
+                let now = std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .map(|d| d.as_secs())
+                    .unwrap_or(0);
+                let refresh_at = next_exp.saturating_sub(5 * 60);
+                if refresh_at > now {
+                    std::thread::sleep(std::time::Duration::from_secs(refresh_at - now));
+                }
+                let result = if let (Some(rt), OAuthFlow::DeviceCode { tenant, client_id }) =
+                    (refresh_token.as_deref(), &flow)
+                {
+                    crate::exchange_ews::oauth::refresh_with_token(
+                        tenant,
+                        client_id,
+                        rt,
+                        allow_invalid_certs,
+                    )
+                } else {
+                    crate::exchange_ews::oauth::acquire(&flow, allow_invalid_certs)
+                };
+                match result {
+                    Ok(tok) => {
+                        client.set_auth(crate::jmap::http::Auth::Bearer {
+                            token: tok.access_token.clone(),
+                        });
+                        if tok.refresh_token.is_some() {
+                            refresh_token = tok.refresh_token.clone();
+                        }
+                        if let Some(new_exp) =
+                            crate::exchange_ews::oauth::decode_jwt_claims(&tok.access_token)
+                                .and_then(|c| c.exp)
+                        {
+                            next_exp = new_exp;
+                        } else {
+                            next_exp = next_exp.saturating_add(50 * 60);
+                        }
+                    }
+                    Err(e) => {
+                        logger.warn(&format!(
+                            "EWS token refresh failed: {e}; sleeping 60s before retry"
+                        ));
+                        std::thread::sleep(std::time::Duration::from_secs(60));
+                    }
+                }
+            }
+        })
+        .ok();
+}
+
+fn run_gc(conn: &Connection) -> Result<(), Error> {
+    let tx = conn.unchecked_transaction()?;
+    db::blobs::gc_orphan_blobs(&tx)?;
+    tx.commit()?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn synthetic_account_id_uses_smtp_for_primary() {
+        assert_eq!(
+            synthetic_account_id("[email protected]", MailboxKind::Primary),
+            "[email protected]"
+        );
+    }
+
+    #[test]
+    fn synthetic_account_id_archive_suffix() {
+        assert_eq!(
+            synthetic_account_id("[email protected]", MailboxKind::Archive),
+            "[email protected]#archive"
+        );
+    }
+
+    #[test]
+    fn synthetic_account_id_public_folders_uses_tenant() {
+        assert_eq!(
+            synthetic_account_id("[email protected]", MailboxKind::PublicFolders),
+            "[email protected]"
+        );
+    }
+
+    #[test]
+    fn basic_against_office365_endpoint_is_refused() {
+        let auth = EwsAuth::Basic {
+            user: "alice@x".to_owned(),
+            password: "p".to_owned(),
+        };
+        let res =
+            enforce_basic_auth_policy(&auth, "https://outlook.office365.com/EWS/Exchange.asmx");
+        assert!(matches!(res, Err(Error::Connection(_))));
+    }
+
+    #[test]
+    fn basic_over_https_on_prem_is_allowed() {
+        let auth = EwsAuth::Basic {
+            user: "alice@x".to_owned(),
+            password: "p".to_owned(),
+        };
+        assert!(
+            enforce_basic_auth_policy(&auth, "https://exchange.example.com/EWS/Exchange.asmx")
+                .is_ok()
+        );
+    }
+
+    #[test]
+    fn basic_over_http_is_refused() {
+        let auth = EwsAuth::Basic {
+            user: "alice".to_owned(),
+            password: "p".to_owned(),
+        };
+        assert!(
+            enforce_basic_auth_policy(&auth, "http://exchange.example.com/EWS/Exchange.asmx")
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn normalise_ews_url_lowercases_host_and_keeps_canonical_path() {
+        assert_eq!(
+            normalise_ews_url("https://OUTLOOK.OFFICE365.COM/EWS/Exchange.asmx"),
+            "https://outlook.office365.com/EWS/Exchange.asmx"
+        );
+        assert_eq!(
+            normalise_ews_url("https://outlook.office365.com:443/ews/exchange.asmx"),
+            "https://outlook.office365.com/EWS/Exchange.asmx"
+        );
+        assert_eq!(
+            normalise_ews_url("https://srv.example.com:8443/EWS/Exchange.asmx"),
+            "https://srv.example.com:8443/EWS/Exchange.asmx"
+        );
+    }
+}

src/sync/import_exchange_ews/folders.rs

@@ -0,0 +1,588 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use rusqlite::{Connection, params};
+
+use crate::db::exchange_ews_ids;
+use crate::error::Error;
+use crate::exchange_ews::EwsClient;
+use crate::exchange_ews::error::EwsError;
+use crate::exchange_ews::parse::{
+    FolderEntry, parse_find_folder_response, parse_folder_inner, parse_get_folder_response,
+    parse_response_messages,
+};
+use crate::exchange_ews::types::{DistinguishedFolderId, FolderClass, MailboxKind};
+use crate::exchange_ews::xml::{
+    FolderRef, FolderShape, Traversal, find_folder_body, get_folder_body,
+};
+use crate::logging::{LEVEL_PROGRESS, Logger};
+use crate::sync::TypeCounts;
+
+pub struct FolderPlan {
+    pub mail: Vec<ClassifiedFolder>,
+    pub calendar: Vec<ClassifiedFolder>,
+    pub contacts: Vec<ClassifiedFolder>,
+    pub well_known_roles: HashMap<String, &'static str>,
+    pub root_folder_id: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassifiedFolder {
+    pub folder: FolderEntry,
+    pub kind: FolderClass,
+}
+
+pub fn plan_folders(
+    client: &EwsClient,
+    url: &str,
+    mailbox_kind: MailboxKind,
+    logger: Logger,
+) -> Result<FolderPlan, EwsError> {
+    let root = mailbox_kind.distinguished_root();
+    let request = find_folder_body(FolderRef::Distinguished(root), Traversal::Deep);
+    let resp = client.call(url, "FindFolder", &request)?;
+    let parsed = parse_find_folder_response(&resp.body)?;
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!("FindFolder returned {} folders", parsed.folders.len());
+    }
+    let mut mail = Vec::new();
+    let mut calendar = Vec::new();
+    let mut contacts = Vec::new();
+    for folder in parsed.folders {
+        let class = FolderClass::from_ipf(&folder.folder_class);
+        match (mailbox_kind, class) {
+            (MailboxKind::Archive, FolderClass::Mail) => {
+                mail.push(ClassifiedFolder {
+                    folder,
+                    kind: class,
+                });
+            }
+            (MailboxKind::Archive, _) => {}
+            (_, FolderClass::Mail) => mail.push(ClassifiedFolder {
+                folder,
+                kind: class,
+            }),
+            (_, FolderClass::Calendar) => {
+                calendar.push(ClassifiedFolder {
+                    folder,
+                    kind: class,
+                });
+            }
+            (_, FolderClass::Contacts) => {
+                contacts.push(ClassifiedFolder {
+                    folder,
+                    kind: class,
+                });
+            }
+            (_, FolderClass::Skipped) => {}
+        }
+    }
+    let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, logger)?;
+    let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind)?;
+    Ok(FolderPlan {
+        mail,
+        calendar,
+        contacts,
+        well_known_roles,
+        root_folder_id,
+    })
+}
+
+fn resolve_well_known_roles(
+    client: &EwsClient,
+    url: &str,
+    mailbox_kind: MailboxKind,
+    logger: Logger,
+) -> Result<HashMap<String, &'static str>, EwsError> {
+    if !matches!(mailbox_kind, MailboxKind::Primary) {
+        return Ok(HashMap::new());
+    }
+    let folders: Vec<FolderRef<'_>> = WELL_KNOWN_ROLES
+        .iter()
+        .map(|(d, _)| FolderRef::Distinguished(*d))
+        .collect();
+    let request = get_folder_body(&folders, FolderShape::IdOnly);
+    let resp = match client.call(url, "GetFolder", &request) {
+        Ok(r) => r,
+        Err(e) => {
+            if logger.enabled(LEVEL_PROGRESS) {
+                eprintln!("GetFolder for distinguished ids failed: {e}; roles will be empty");
+            }
+            return Ok(HashMap::new());
+        }
+    };
+    let messages =
+        parse_response_messages(&resp.body, b"GetFolderResponseMessage").unwrap_or_default();
+    let mut map: HashMap<String, &'static str> = HashMap::new();
+    for (msg, (_, role)) in messages.into_iter().zip(WELL_KNOWN_ROLES.iter()) {
+        if !msg.success {
+            continue;
+        }
+        let Ok(Some(entry)) = parse_folder_inner(&msg.inner_xml) else {
+            continue;
+        };
+        if entry.folder_id.id.is_empty() {
+            continue;
+        }
+        if let Some(r) = role {
+            map.insert(entry.folder_id.id, r);
+        }
+    }
+    Ok(map)
+}
+
+fn resolve_root_folder_id(
+    client: &EwsClient,
+    url: &str,
+    mailbox_kind: MailboxKind,
+) -> Result<String, EwsError> {
+    let root = mailbox_kind.distinguished_root();
+    let body = get_folder_body(&[FolderRef::Distinguished(root)], FolderShape::IdOnly);
+    let resp = client.call(url, "GetFolder", &body)?;
+    let entries = parse_get_folder_response(&resp.body)?;
+    Ok(entries
+        .into_iter()
+        .next()
+        .map(|e| e.folder_id.id)
+        .unwrap_or_default())
+}
+
+const WELL_KNOWN_ROLES: &[(DistinguishedFolderId, Option<&'static str>)] = &[
+    (DistinguishedFolderId::Inbox, Some("inbox")),
+    (DistinguishedFolderId::SentItems, Some("sent")),
+    (DistinguishedFolderId::Drafts, Some("drafts")),
+    (DistinguishedFolderId::DeletedItems, Some("trash")),
+    (DistinguishedFolderId::JunkEmail, Some("junk")),
+    (DistinguishedFolderId::Archive, Some("archive")),
+    (DistinguishedFolderId::Outbox, None),
+    (DistinguishedFolderId::ConversationHistory, None),
+];
+
+pub fn reconcile(
+    conn: &mut Connection,
+    source_id: i64,
+    plan: &FolderPlan,
+    mailbox_counts: &mut TypeCounts,
+    calendar_counts: &mut TypeCounts,
+    addressbook_counts: &mut TypeCounts,
+    _logger: Logger,
+) -> Result<(), Error> {
+    let local_mailbox: HashMap<String, exchange_ews_ids::FolderRow> =
+        exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::MAILBOX)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let local_calendar: HashMap<String, exchange_ews_ids::FolderRow> =
+        exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::CALENDAR)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let local_addressbook: HashMap<String, exchange_ews_ids::FolderRow> =
+        exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::ADDRESS_BOOK)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+
+    let mut id_to_local: HashMap<String, i64> = HashMap::new();
+    for (id, row) in local_mailbox
+        .iter()
+        .chain(local_calendar.iter())
+        .chain(local_addressbook.iter())
+    {
+        id_to_local.insert(id.clone(), row.local_id);
+    }
+
+    let ordered_mail = order_by_parent(&plan.mail);
+    let ordered_calendar = order_by_parent(&plan.calendar);
+    let ordered_contacts = order_by_parent(&plan.contacts);
+
+    let mut server_ids_mail: Vec<String> = Vec::new();
+    let mut server_ids_calendar: Vec<String> = Vec::new();
+    let mut server_ids_addressbook: Vec<String> = Vec::new();
+
+    for folder in &ordered_mail {
+        server_ids_mail.push(folder.folder.folder_id.id.clone());
+        let role = plan
+            .well_known_roles
+            .get(&folder.folder.folder_id.id)
+            .copied();
+        let mut ctx = UpsertCtx {
+            conn,
+            source_id,
+            id_to_local: &mut id_to_local,
+            local: &local_mailbox,
+            counts: mailbox_counts,
+            root_folder_id: &plan.root_folder_id,
+        };
+        upsert_mailbox(&mut ctx, folder, role)?;
+    }
+    for folder in &ordered_calendar {
+        server_ids_calendar.push(folder.folder.folder_id.id.clone());
+        let mut ctx = UpsertCtx {
+            conn,
+            source_id,
+            id_to_local: &mut id_to_local,
+            local: &local_calendar,
+            counts: calendar_counts,
+            root_folder_id: &plan.root_folder_id,
+        };
+        upsert_calendar(&mut ctx, folder)?;
+    }
+    for folder in &ordered_contacts {
+        server_ids_addressbook.push(folder.folder.folder_id.id.clone());
+        let mut ctx = UpsertCtx {
+            conn,
+            source_id,
+            id_to_local: &mut id_to_local,
+            local: &local_addressbook,
+            counts: addressbook_counts,
+            root_folder_id: &plan.root_folder_id,
+        };
+        upsert_address_book(&mut ctx, folder)?;
+    }
+
+    delete_vanished(
+        conn,
+        source_id,
+        exchange_ews_ids::MAILBOX,
+        "mailboxes",
+        &local_mailbox,
+        &server_ids_mail,
+        mailbox_counts,
+    )?;
+    delete_vanished(
+        conn,
+        source_id,
+        exchange_ews_ids::CALENDAR,
+        "calendars",
+        &local_calendar,
+        &server_ids_calendar,
+        calendar_counts,
+    )?;
+    delete_vanished(
+        conn,
+        source_id,
+        exchange_ews_ids::ADDRESS_BOOK,
+        "address_books",
+        &local_addressbook,
+        &server_ids_addressbook,
+        addressbook_counts,
+    )?;
+    Ok(())
+}
+
+fn order_by_parent(folders: &[ClassifiedFolder]) -> Vec<&ClassifiedFolder> {
+    let by_id: HashMap<&str, &ClassifiedFolder> = folders
+        .iter()
+        .map(|f| (f.folder.folder_id.id.as_str(), f))
+        .collect();
+    let mut depths: HashMap<&str, usize> = HashMap::new();
+    fn depth_of<'a>(
+        id: &'a str,
+        by_id: &HashMap<&'a str, &'a ClassifiedFolder>,
+        memo: &mut HashMap<&'a str, usize>,
+        seen: &mut std::collections::HashSet<&'a str>,
+    ) -> usize {
+        if let Some(d) = memo.get(id) {
+            return *d;
+        }
+        if !seen.insert(id) {
+            return 0;
+        }
+        let d = match by_id.get(id).and_then(|f| f.folder.parent_id.as_deref()) {
+            Some(parent) if by_id.contains_key(parent) => 1 + depth_of(parent, by_id, memo, seen),
+            _ => 0,
+        };
+        memo.insert(id, d);
+        d
+    }
+    for f in folders {
+        let id = f.folder.folder_id.id.as_str();
+        let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
+        let d = depth_of(id, &by_id, &mut depths, &mut seen);
+        depths.insert(id, d);
+    }
+    let mut out: Vec<&ClassifiedFolder> = folders.iter().collect();
+    out.sort_by_key(|f| {
+        depths
+            .get(f.folder.folder_id.id.as_str())
+            .copied()
+            .unwrap_or(0)
+    });
+    out
+}
+
+struct UpsertCtx<'a> {
+    conn: &'a mut Connection,
+    source_id: i64,
+    id_to_local: &'a mut HashMap<String, i64>,
+    local: &'a HashMap<String, exchange_ews_ids::FolderRow>,
+    counts: &'a mut TypeCounts,
+    root_folder_id: &'a str,
+}
+
+fn upsert_mailbox(
+    ctx: &mut UpsertCtx<'_>,
+    folder: &ClassifiedFolder,
+    role: Option<&'static str>,
+) -> Result<(), Error> {
+    let parent_local = parent_local_id(folder, ctx.id_to_local, ctx.root_folder_id);
+    let tx = ctx
+        .conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let existing = ctx.local.get(&folder.folder.folder_id.id);
+    let local_id = if let Some(row) = existing {
+        tx.execute(
+            "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3 WHERE id = ?4",
+            params![folder.folder.display_name, parent_local, role, row.local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        exchange_ews_ids::update_change_key(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.fetched += 1;
+        row.local_id
+    } else {
+        tx.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) \
+             VALUES (?1, ?2, ?3, 0, 1)",
+            params![folder.folder.display_name, parent_local, role],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let new_id = tx.last_insert_rowid();
+        exchange_ews_ids::insert(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            folder.folder.parent_id.as_deref().unwrap_or(""),
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+            new_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.created += 1;
+        new_id
+    };
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    ctx.id_to_local
+        .insert(folder.folder.folder_id.id.clone(), local_id);
+    Ok(())
+}
+
+fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> {
+    let tx = ctx
+        .conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let existing = ctx.local.get(&folder.folder.folder_id.id);
+    let local_id = if let Some(row) = existing {
+        tx.execute(
+            "UPDATE calendars SET name = ?1 WHERE id = ?2",
+            params![folder.folder.display_name, row.local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        exchange_ews_ids::update_change_key(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.fetched += 1;
+        row.local_id
+    } else {
+        tx.execute(
+            "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible) \
+             VALUES (?1, 0, 1, 1)",
+            params![folder.folder.display_name],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let new_id = tx.last_insert_rowid();
+        exchange_ews_ids::insert(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            folder.folder.parent_id.as_deref().unwrap_or(""),
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+            new_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.created += 1;
+        new_id
+    };
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    ctx.id_to_local
+        .insert(folder.folder.folder_id.id.clone(), local_id);
+    Ok(())
+}
+
+fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> {
+    let tx = ctx
+        .conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let existing = ctx.local.get(&folder.folder.folder_id.id);
+    let local_id = if let Some(row) = existing {
+        tx.execute(
+            "UPDATE address_books SET name = ?1 WHERE id = ?2",
+            params![folder.folder.display_name, row.local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        exchange_ews_ids::update_change_key(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.fetched += 1;
+        row.local_id
+    } else {
+        tx.execute(
+            "INSERT INTO address_books (name, sort_order, is_subscribed) \
+             VALUES (?1, 0, 1)",
+            params![folder.folder.display_name],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let new_id = tx.last_insert_rowid();
+        exchange_ews_ids::insert(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            folder.folder.parent_id.as_deref().unwrap_or(""),
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+            new_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.created += 1;
+        new_id
+    };
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    ctx.id_to_local
+        .insert(folder.folder.folder_id.id.clone(), local_id);
+    Ok(())
+}
+
+fn parent_local_id(
+    folder: &ClassifiedFolder,
+    id_to_local: &HashMap<String, i64>,
+    root_folder_id: &str,
+) -> Option<i64> {
+    let parent_id = folder.folder.parent_id.as_deref()?;
+    if parent_id == root_folder_id || parent_id.is_empty() {
+        return None;
+    }
+    id_to_local.get(parent_id).copied()
+}
+
+fn delete_vanished(
+    conn: &mut Connection,
+    source_id: i64,
+    type_name: &str,
+    table: &str,
+    local: &HashMap<String, exchange_ews_ids::FolderRow>,
+    server_ids: &[String],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let server_set: std::collections::HashSet<&str> =
+        server_ids.iter().map(String::as_str).collect();
+    let mut vanished: Vec<(&str, &exchange_ews_ids::FolderRow)> = local
+        .iter()
+        .filter(|(id, _)| !server_set.contains(id.as_str()))
+        .map(|(id, row)| (id.as_str(), row))
+        .collect();
+    if type_name == exchange_ews_ids::MAILBOX {
+        vanished.sort_by_key(|(_, row)| std::cmp::Reverse(folder_depth(row, local)));
+    }
+    for (item_id, row) in vanished {
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        let result = tx.execute(
+            &format!("DELETE FROM {table} WHERE id = ?1"),
+            params![row.local_id],
+        );
+        match result {
+            Ok(_) => {
+                exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id)
+                    .map_err(|e| Error::Partial(e.to_string()))?;
+                tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                counts.deleted += 1;
+            }
+            Err(_) => {
+                let _ = tx.rollback();
+                counts.failed += 1;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn folder_depth(
+    row: &exchange_ews_ids::FolderRow,
+    local: &HashMap<String, exchange_ews_ids::FolderRow>,
+) -> usize {
+    let mut depth = 0;
+    let mut cursor = row.folder_id.as_str();
+    let mut visited: std::collections::HashSet<&str> = std::collections::HashSet::new();
+    while !cursor.is_empty() && visited.insert(cursor) {
+        match local.get(cursor) {
+            Some(parent) => {
+                depth += 1;
+                cursor = parent.folder_id.as_str();
+            }
+            None => break,
+        }
+    }
+    depth
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn row(id: &str, parent_ews_id: &str, local_id: i64) -> exchange_ews_ids::FolderRow {
+        exchange_ews_ids::FolderRow {
+            item_id: id.to_owned(),
+            folder_id: parent_ews_id.to_owned(),
+            change_key: String::new(),
+            local_id,
+        }
+    }
+
+    #[test]
+    fn folder_depth_walks_parent_chain() {
+        let mut local: HashMap<String, exchange_ews_ids::FolderRow> = HashMap::new();
+        local.insert("ROOT".to_owned(), row("ROOT", "", 1));
+        local.insert("CHILD".to_owned(), row("CHILD", "ROOT", 2));
+        local.insert("GRAND".to_owned(), row("GRAND", "CHILD", 3));
+        assert_eq!(folder_depth(local.get("ROOT").unwrap(), &local), 0);
+        assert_eq!(folder_depth(local.get("CHILD").unwrap(), &local), 1);
+        assert_eq!(folder_depth(local.get("GRAND").unwrap(), &local), 2);
+    }
+
+    #[test]
+    fn folder_depth_handles_unknown_parent() {
+        let mut local: HashMap<String, exchange_ews_ids::FolderRow> = HashMap::new();
+        local.insert("X".to_owned(), row("X", "ORPHAN", 1));
+        assert_eq!(folder_depth(local.get("X").unwrap(), &local), 0);
+    }
+
+    #[test]
+    fn folder_depth_stops_on_cycle() {
+        let mut local: HashMap<String, exchange_ews_ids::FolderRow> = HashMap::new();
+        local.insert("A".to_owned(), row("A", "B", 1));
+        local.insert("B".to_owned(), row("B", "A", 2));
+        let d = folder_depth(local.get("A").unwrap(), &local);
+        assert!(d <= 2);
+    }
+}

src/sync/import_exchange_ews/items.rs

@@ -0,0 +1,469 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use crate::error::Error;
+use crate::exchange_ews::EwsClient;
+use crate::exchange_ews::error::EwsError;
+use crate::exchange_ews::parse::{
+    SyncChange, parse_find_item_response, parse_response_messages, parse_sync_folder_items_response,
+};
+use crate::exchange_ews::types::{FolderId, ItemId, ResponseCode};
+use crate::exchange_ews::xml::{
+    FolderRef, ItemShape, Traversal, find_item_body, get_item_body, sync_folder_items_body,
+};
+use crate::logging::Logger;
+
+pub struct ItemRunCtx<'a> {
+    pub client: &'a EwsClient,
+    pub url: &'a str,
+    pub source_id: i64,
+    pub batch_size: usize,
+    pub attachment_batch: usize,
+    pub connections: usize,
+    pub use_syncfolderitems: bool,
+    pub logger: Logger,
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumeratedItem {
+    pub element: String,
+    pub id: ItemId,
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumerationOutcome {
+    pub items: Vec<EnumeratedItem>,
+    pub mode: EnumerationMode,
+}
+
+#[derive(Debug, Clone)]
+pub enum EnumerationMode {
+    Full,
+    Delta {
+        deletions: Vec<String>,
+        new_sync_state: String,
+    },
+}
+
+pub fn enumerate_folder(
+    ctx: &ItemRunCtx<'_>,
+    folder: &FolderId,
+    prior_sync_state: Option<&str>,
+) -> Result<EnumerationOutcome, EwsError> {
+    if ctx.use_syncfolderitems
+        && let Some(state) = prior_sync_state
+        && !state.is_empty()
+        && let Some(outcome) = try_sync_folder_items(ctx, folder, state)?
+    {
+        return Ok(outcome);
+    }
+    enumerate_via_find_item(ctx, folder).map(|items| EnumerationOutcome {
+        items,
+        mode: EnumerationMode::Full,
+    })
+}
+
+fn sync_fallback_codes(code: &ResponseCode) -> bool {
+    matches!(
+        code,
+        ResponseCode::InvalidIdMalformed | ResponseCode::AccessDenied | ResponseCode::Other(_)
+    )
+}
+
+fn try_sync_folder_items(
+    ctx: &ItemRunCtx<'_>,
+    folder: &FolderId,
+    prior_state: &str,
+) -> Result<Option<EnumerationOutcome>, EwsError> {
+    let mut sync_state = prior_state.to_owned();
+    let mut items: Vec<EnumeratedItem> = Vec::new();
+    let mut deletions: Vec<String> = Vec::new();
+    let mut iters = 0;
+    loop {
+        let body = sync_folder_items_body(folder, &sync_state, 512);
+        let resp = match ctx.client.call(ctx.url, "SyncFolderItems", &body) {
+            Ok(r) => r,
+            Err(EwsError::SoapFault {
+                code: ResponseCode::InvalidSyncStateData,
+                ..
+            }) => {
+                return Ok(None);
+            }
+            Err(EwsError::SoapFault { code, .. }) if sync_fallback_codes(&code) => {
+                return Ok(None);
+            }
+            Err(e) => return Err(e),
+        };
+        let parsed = parse_sync_folder_items_response(&resp.body)?;
+        for change in parsed.changes {
+            match change {
+                SyncChange::Create { id, element } | SyncChange::Update { id, element } => {
+                    items.push(EnumeratedItem { element, id });
+                }
+                SyncChange::Delete { id } => {
+                    deletions.push(id.id);
+                }
+                SyncChange::ReadFlagChange { id, .. } => {
+                    items.push(EnumeratedItem {
+                        element: "Message".to_owned(),
+                        id,
+                    });
+                }
+            }
+        }
+        sync_state = parsed.sync_state;
+        if !parsed.more {
+            break;
+        }
+        iters += 1;
+        if iters > 200 {
+            break;
+        }
+    }
+    Ok(Some(EnumerationOutcome {
+        items,
+        mode: EnumerationMode::Delta {
+            deletions,
+            new_sync_state: sync_state,
+        },
+    }))
+}
+
+fn enumerate_via_find_item(
+    ctx: &ItemRunCtx<'_>,
+    folder: &FolderId,
+) -> Result<Vec<EnumeratedItem>, EwsError> {
+    let mut items: Vec<EnumeratedItem> = Vec::new();
+    let mut offset: u32 = 0;
+    let page_size: u32 = 500;
+    loop {
+        let body = find_item_body(
+            FolderRef::Concrete(folder),
+            Traversal::Shallow,
+            offset,
+            page_size,
+        );
+        let resp = ctx.client.call(ctx.url, "FindItem", &body)?;
+        let parsed = parse_find_item_response(&resp.body)?;
+        let returned = parsed.items.len() as u32;
+        for entry in parsed.items {
+            items.push(EnumeratedItem {
+                element: entry.element,
+                id: entry.id,
+            });
+        }
+        if !parsed.more {
+            break;
+        }
+        if returned == 0 {
+            break;
+        }
+        offset = offset.saturating_add(page_size);
+        if offset > 1_000_000 {
+            break;
+        }
+    }
+    Ok(items)
+}
+
+#[derive(Debug, Clone)]
+pub struct DiffPlan {
+    pub new: Vec<ItemId>,
+    pub vanished: Vec<(String, i64)>,
+    pub present_changed: Vec<(ItemId, i64)>,
+    pub present_unchanged: Vec<(String, i64)>,
+}
+
+pub fn plan_for(
+    outcome: &EnumerationOutcome,
+    local: &[crate::db::exchange_ews_ids::ItemRow],
+) -> DiffPlan {
+    match &outcome.mode {
+        EnumerationMode::Full => diff(&outcome.items, local),
+        EnumerationMode::Delta { deletions, .. } => diff_delta(&outcome.items, deletions, local),
+    }
+}
+
+fn diff_delta(
+    server_changes: &[EnumeratedItem],
+    deletions: &[String],
+    local: &[crate::db::exchange_ews_ids::ItemRow],
+) -> DiffPlan {
+    let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> =
+        local.iter().map(|r| (r.item_id.as_str(), r)).collect();
+    let mut plan = DiffPlan {
+        new: Vec::new(),
+        vanished: Vec::new(),
+        present_changed: Vec::new(),
+        present_unchanged: Vec::new(),
+    };
+    for s in server_changes {
+        match local_map.get(s.id.id.as_str()) {
+            None => plan.new.push(s.id.clone()),
+            Some(row) => plan.present_changed.push((s.id.clone(), row.local_id)),
+        }
+    }
+    for d in deletions {
+        if let Some(row) = local_map.get(d.as_str()) {
+            plan.vanished.push((d.clone(), row.local_id));
+        }
+    }
+    plan
+}
+
+pub fn diff(server: &[EnumeratedItem], local: &[crate::db::exchange_ews_ids::ItemRow]) -> DiffPlan {
+    let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> =
+        local.iter().map(|r| (r.item_id.as_str(), r)).collect();
+    let server_map: HashMap<&str, &EnumeratedItem> =
+        server.iter().map(|s| (s.id.id.as_str(), s)).collect();
+    let mut plan = DiffPlan {
+        new: Vec::new(),
+        vanished: Vec::new(),
+        present_changed: Vec::new(),
+        present_unchanged: Vec::new(),
+    };
+    for s in server {
+        match local_map.get(s.id.id.as_str()) {
+            None => plan.new.push(s.id.clone()),
+            Some(row) => {
+                if s.id.change_key.is_empty() || s.id.change_key == row.change_key {
+                    plan.present_unchanged
+                        .push((row.item_id.clone(), row.local_id));
+                } else {
+                    plan.present_changed.push((s.id.clone(), row.local_id));
+                }
+            }
+        }
+    }
+    for row in local {
+        if !server_map.contains_key(row.item_id.as_str()) {
+            plan.vanished.push((row.item_id.clone(), row.local_id));
+        }
+    }
+    plan
+}
+
+pub struct GetItemBatchOutcome {
+    pub messages: Vec<crate::exchange_ews::parse::ResponseMessage>,
+    pub failed_items: u64,
+}
+
+fn is_per_batch_fault(err: &EwsError) -> bool {
+    matches!(
+        err,
+        EwsError::SoapFault { .. } | EwsError::HttpStatus { .. } | EwsError::Malformed(_)
+    )
+}
+
+pub fn get_items(
+    ctx: &ItemRunCtx<'_>,
+    shape: ItemShape,
+    ids: &[ItemId],
+) -> Result<GetItemBatchOutcome, EwsError> {
+    let batch = ctx.batch_size.max(1);
+    let workers = ctx.connections.clamp(1, 8);
+    let mut failed_items: u64 = 0;
+    if workers <= 1 || ids.len() <= batch {
+        let mut all = Vec::new();
+        for chunk in ids.chunks(batch) {
+            let body = get_item_body(shape, chunk);
+            match ctx.client.call(ctx.url, "GetItem", &body) {
+                Ok(resp) => match parse_response_messages(&resp.body, b"GetItemResponseMessage") {
+                    Ok(mut msgs) => all.append(&mut msgs),
+                    Err(e) if is_per_batch_fault(&e) => {
+                        ctx.logger.warn(&format!(
+                            "GetItem batch parse failed ({e}); {} ids left for retry next run",
+                            chunk.len()
+                        ));
+                        failed_items += chunk.len() as u64;
+                    }
+                    Err(e) => return Err(e),
+                },
+                Err(e) if is_per_batch_fault(&e) => {
+                    ctx.logger.warn(&format!(
+                        "GetItem batch failed ({e}); {} ids left for retry next run",
+                        chunk.len()
+                    ));
+                    failed_items += chunk.len() as u64;
+                }
+                Err(e) => return Err(e),
+            }
+        }
+        return Ok(GetItemBatchOutcome {
+            messages: all,
+            failed_items,
+        });
+    }
+    let client = ctx.client.clone();
+    let url = ctx.url.to_owned();
+    type BatchResult = (
+        usize,
+        Result<Vec<crate::exchange_ews::parse::ResponseMessage>, EwsError>,
+    );
+    let pool: crate::sync::import_jmap::pool::Pool<Vec<ItemId>, BatchResult> =
+        crate::sync::import_jmap::pool::Pool::new(workers, move |chunk: Vec<ItemId>| {
+            let body = get_item_body(shape, &chunk);
+            let n = chunk.len();
+            let result = match client.call(&url, "GetItem", &body) {
+                Ok(resp) => parse_response_messages(&resp.body, b"GetItemResponseMessage"),
+                Err(e) => Err(e),
+            };
+            (n, result)
+        });
+    let mut submitted = 0usize;
+    for chunk in ids.chunks(batch) {
+        pool.submit(chunk.to_vec());
+        submitted += 1;
+    }
+    let mut all = Vec::new();
+    let mut abort_err: Option<EwsError> = None;
+    for _ in 0..submitted {
+        match pool.results().recv() {
+            Ok((_, Ok(mut msgs))) => all.append(&mut msgs),
+            Ok((n, Err(e))) => {
+                if is_per_batch_fault(&e) {
+                    ctx.logger.warn(&format!(
+                        "GetItem batch failed ({e}); {n} ids left for retry next run"
+                    ));
+                    failed_items += n as u64;
+                } else if abort_err.is_none() {
+                    abort_err = Some(e);
+                }
+            }
+            Err(_) => break,
+        }
+    }
+    if let Some(e) = abort_err {
+        return Err(e);
+    }
+    Ok(GetItemBatchOutcome {
+        messages: all,
+        failed_items,
+    })
+}
+
+pub fn delete_vanished(
+    conn: &mut rusqlite::Connection,
+    source_id: i64,
+    type_name: &str,
+    table: &str,
+    vanished: &[(String, i64)],
+    counts: &mut crate::sync::TypeCounts,
+) -> Result<(), Error> {
+    use rusqlite::params;
+    for (item_id, local_id) in vanished {
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        match tx.execute(
+            &format!("DELETE FROM {table} WHERE id = ?1"),
+            params![local_id],
+        ) {
+            Ok(_) => {
+                crate::db::exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id)
+                    .map_err(|e| Error::Partial(e.to_string()))?;
+                tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                counts.deleted += 1;
+            }
+            Err(_) => {
+                let _ = tx.rollback();
+                counts.failed += 1;
+            }
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn diff_delta_only_yields_changes_from_server_response() {
+        use crate::db::exchange_ews_ids::ItemRow;
+        let outcome = EnumerationOutcome {
+            items: vec![EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("A", "ck-2"),
+            }],
+            mode: EnumerationMode::Delta {
+                deletions: vec!["Z".to_owned()],
+                new_sync_state: "STATE2".to_owned(),
+            },
+        };
+        let local = vec![
+            ItemRow {
+                item_id: "A".to_owned(),
+                change_key: "ck-1".to_owned(),
+                local_id: 1,
+            },
+            ItemRow {
+                item_id: "B".to_owned(),
+                change_key: "ck-1".to_owned(),
+                local_id: 2,
+            },
+            ItemRow {
+                item_id: "Z".to_owned(),
+                change_key: "ck-9".to_owned(),
+                local_id: 99,
+            },
+        ];
+        let plan = plan_for(&outcome, &local);
+        assert!(plan.new.is_empty());
+        assert_eq!(plan.present_changed.len(), 1);
+        assert_eq!(plan.present_changed[0].0.id, "A");
+        assert_eq!(plan.vanished.len(), 1);
+        assert_eq!(plan.vanished[0].0, "Z");
+        assert_eq!(plan.present_unchanged.len(), 0);
+    }
+
+    #[test]
+    fn diff_splits_into_new_unchanged_changed_vanished() {
+        use crate::db::exchange_ews_ids::ItemRow;
+        let server = vec![
+            EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("A", "ck-1"),
+            },
+            EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("B", "ck-1"),
+            },
+            EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("C", "ck-2"),
+            },
+        ];
+        let local = vec![
+            ItemRow {
+                item_id: "A".to_owned(),
+                change_key: "ck-1".to_owned(),
+                local_id: 1,
+            },
+            ItemRow {
+                item_id: "B".to_owned(),
+                change_key: "ck-0".to_owned(),
+                local_id: 2,
+            },
+            ItemRow {
+                item_id: "Z".to_owned(),
+                change_key: "ck-9".to_owned(),
+                local_id: 99,
+            },
+        ];
+        let plan = diff(&server, &local);
+        assert_eq!(plan.new.len(), 1);
+        assert_eq!(plan.new[0].id, "C");
+        assert_eq!(plan.present_unchanged.len(), 1);
+        assert_eq!(plan.present_unchanged[0].0, "A");
+        assert_eq!(plan.present_changed.len(), 1);
+        assert_eq!(plan.present_changed[0].0.id, "B");
+        assert_eq!(plan.vanished.len(), 1);
+        assert_eq!(plan.vanished[0].0, "Z");
+    }
+}

src/sync/import_exchange_ews/messages.rs

@@ -0,0 +1,262 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD;
+use rusqlite::{Connection, params};
+use serde_json::{Value, json};
+
+use crate::db::{blobs, exchange_ews_ids};
+use crate::error::Error;
+use crate::exchange_ews::parse::{MessageItem, parse_message_item};
+use crate::exchange_ews::types::ItemId;
+use crate::exchange_ews::xml::ItemShape;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+use crate::sync::emailmeta::email_meta_from_blob;
+use crate::sync::keys::index_to_json;
+
+use super::folders::FolderPlan;
+use super::items::{
+    EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
+};
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    plan: &FolderPlan,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for folder in &plan.mail {
+        let folder_id = &folder.folder.folder_id;
+        let local_folder_id = match exchange_ews_ids::local_for_item(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            &folder_id.id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        {
+            Some(id) => id,
+            None => continue,
+        };
+        if let Err(e) = reconcile_one_folder(conn, ctx, folder_id, local_folder_id, counts) {
+            ctx.logger
+                .warn(&format!("email folder {} failed: {}", folder_id.id, e));
+            counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn reconcile_one_folder(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    folder: &crate::exchange_ews::types::FolderId,
+    local_folder_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let prior = exchange_ews_ids::get_sync_state(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::MAILBOX,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?;
+    let local =
+        exchange_ews_ids::items_in_folder(conn, ctx.source_id, exchange_ews_ids::EMAIL, &folder.id)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let plan = plan_for(&outcome, &local);
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS folder {}: new={} changed={} vanished={} unchanged={}",
+            folder.id,
+            plan.new.len(),
+            plan.present_changed.len(),
+            plan.vanished.len(),
+            plan.present_unchanged.len()
+        );
+    }
+    let mut to_fetch: Vec<ItemId> = plan.new.clone();
+    for (id, _local_id) in &plan.present_changed {
+        to_fetch.push(id.clone());
+    }
+    if !to_fetch.is_empty() {
+        let outcome = get_items(ctx, ItemShape::Message, &to_fetch).map_err(Error::from)?;
+        counts.failed += outcome.failed_items;
+        for msg in outcome.messages {
+            if !msg.success {
+                if matches!(
+                    msg.response_code,
+                    crate::exchange_ews::types::ResponseCode::ItemNotFound
+                ) {
+                    counts.skipped += 1;
+                } else {
+                    counts.failed += 1;
+                    ctx.logger.warn(&format!(
+                        "GetItem (message) error: {} {}",
+                        msg.response_code, msg.message_text
+                    ));
+                }
+                continue;
+            }
+            let parsed = parse_message_item(&msg.inner_xml).map_err(Error::from)?;
+            if parsed.id.id.is_empty() {
+                counts.failed += 1;
+                continue;
+            }
+            let existing = plan
+                .present_changed
+                .iter()
+                .find(|(id, _)| id.id == parsed.id.id)
+                .map(|(_, local)| *local);
+            apply_message(
+                conn,
+                ctx,
+                &parsed,
+                local_folder_id,
+                &folder.id,
+                existing,
+                counts,
+            )?;
+        }
+    }
+    delete_vanished(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::EMAIL,
+        "emails",
+        &plan.vanished,
+        counts,
+    )?;
+    if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode {
+        exchange_ews_ids::set_sync_state(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            &folder.id,
+            new_sync_state,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn apply_message(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    item: &MessageItem,
+    local_folder_id: i64,
+    folder_id: &str,
+    existing_local_id: Option<i64>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let mime_b64 = match item.mime_content.as_ref() {
+        Some(s) => s.replace(['\n', '\r', ' ', '\t'], ""),
+        None => {
+            counts.skipped += 1;
+            ctx.logger.warn(&format!(
+                "message {} has no MimeContent; skipping",
+                item.id.id
+            ));
+            return Ok(());
+        }
+    };
+    let bytes = match STANDARD.decode(mime_b64.as_bytes()) {
+        Ok(b) => b,
+        Err(e) => {
+            counts.failed += 1;
+            ctx.logger.warn(&format!(
+                "message {} MimeContent base64 decode failed: {e}",
+                item.id.id
+            ));
+            return Ok(());
+        }
+    };
+    let (idx, _date_header) = email_meta_from_blob(&bytes);
+    let message_match = index_to_json(&idx);
+    let received_at = item
+        .date_time_received
+        .clone()
+        .unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned());
+    let keywords = keyword_array(item);
+    let mailbox_ids = json!([local_folder_id]).to_string();
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let blob_id = blobs::intern_blob(&tx, &bytes).map_err(|e| Error::Partial(e.to_string()))?;
+    match existing_local_id {
+        Some(id) => {
+            tx.execute(
+                "UPDATE emails SET blob_id = ?1, received_at = ?2, mailbox_ids = ?3, \
+                 keywords = ?4, message_match = ?5 WHERE id = ?6",
+                params![
+                    blob_id,
+                    received_at,
+                    mailbox_ids,
+                    keywords,
+                    message_match,
+                    id
+                ],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            exchange_ews_ids::update_change_key(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::EMAIL,
+                &item.id.id,
+                &item.id.change_key,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.fetched += 1;
+        }
+        None => {
+            tx.execute(
+                "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) \
+                 VALUES (?1, ?2, ?3, ?4, ?5)",
+                params![blob_id, received_at, mailbox_ids, keywords, message_match],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            let new_id = tx.last_insert_rowid();
+            exchange_ews_ids::insert(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::EMAIL,
+                folder_id,
+                &item.id.id,
+                &item.id.change_key,
+                new_id,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.created += 1;
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}
+
+fn keyword_array(item: &MessageItem) -> String {
+    let mut kws: Vec<String> = Vec::new();
+    if matches!(item.is_read, Some(true)) {
+        kws.push("$seen".to_owned());
+    }
+    if matches!(item.is_draft, Some(true)) {
+        kws.push("$draft".to_owned());
+    }
+    if matches!(item.is_read_receipt_requested, Some(true)) {
+        kws.push("$notified".to_owned());
+    }
+    if matches!(item.flag_status.as_deref(), Some("Flagged")) {
+        kws.push("$flagged".to_owned());
+    }
+    for cat in &item.categories {
+        kws.push(cat.to_ascii_lowercase());
+    }
+    let value: Value = Value::Array(kws.into_iter().map(Value::String).collect());
+    value.to_string()
+}

src/sync/import_exchange_graph.rs

@@ -0,0 +1,13 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod calendar;
+pub mod contacts;
+pub mod coordinator;
+pub mod folders;
+pub mod messages;
+
+pub use coordinator::{GraphAuth, GraphImportConfig, run};

src/sync/import_exchange_graph/calendar.rs

@@ -0,0 +1,443 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use rusqlite::{Connection, Transaction, params};
+use serde_json::{Value, json};
+
+use crate::db::exchange_graph_ids;
+use crate::error::Error;
+use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC};
+use crate::exchange_graph::calendar_map::{
+    ConvertedEvent, EventType, classify_event_type, convert_event,
+};
+use crate::exchange_graph::error::GraphError;
+use crate::exchange_graph::types::EventBodyFormat;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+use crate::sync::import_jmap::pool::Pool;
+
+use super::coordinator::{CHUNK_SIZE, GraphCoordinator};
+use super::folders::CalendarFolder;
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    calendars: &[CalendarFolder],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let local: HashMap<String, i64> =
+        exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?;
+    let mut server_total: std::collections::HashSet<String> = std::collections::HashSet::new();
+    let mut any_failure = false;
+
+    for cal in calendars {
+        let url = ctx.endpoints.calendar_events_ids(&cal.graph_id, ctx.top);
+        let stubs = match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) {
+            Ok(v) => v,
+            Err(e) => {
+                ctx.logger
+                    .warn(&format!("calendar {} stub fetch failed: {e}", cal.graph_id));
+                counts.failed += 1;
+                any_failure = true;
+                continue;
+            }
+        };
+        let mut want_ids: Vec<String> = Vec::new();
+        let mut occurrence_count = 0usize;
+        for stub in &stubs {
+            match classify_event_type(stub) {
+                EventType::Occurrence => occurrence_count += 1,
+                _ => {
+                    if let Some(id) = stub.get("id").and_then(Value::as_str) {
+                        server_total.insert(id.to_owned());
+                        if !local.contains_key(id) {
+                            want_ids.push(id.to_owned());
+                        }
+                    }
+                }
+            }
+        }
+        if ctx.logger.enabled(LEVEL_PROGRESS) {
+            eprintln!(
+                "graph calendar {} events: stubs={} new={} occurrences_skipped={}",
+                cal.graph_id,
+                stubs.len(),
+                want_ids.len(),
+                occurrence_count
+            );
+        }
+        if want_ids.is_empty() {
+            continue;
+        }
+        let fetched = fetch_events(ctx, &want_ids);
+        let mut masters: Vec<(String, ConvertedEvent)> = Vec::new();
+        let mut exceptions: Vec<(String, ConvertedEvent)> = Vec::new();
+        for (graph_id, result) in fetched {
+            match result {
+                Ok(raw) => match convert_event(&raw, None) {
+                    Ok(c) => match c.event_type {
+                        EventType::Exception => exceptions.push((graph_id, c)),
+                        _ => masters.push((graph_id, c)),
+                    },
+                    Err(e) => {
+                        counts.failed += 1;
+                        ctx.logger
+                            .warn(&format!("graph event {graph_id} convert failed: {e}"));
+                    }
+                },
+                Err(GraphError::Vanished) => counts.skipped += 1,
+                Err(e) => {
+                    counts.failed += 1;
+                    ctx.logger
+                        .warn(&format!("graph event {graph_id} fetch failed: {e}"));
+                }
+            }
+        }
+
+        let mut master_by_graph_id: HashMap<String, ConvertedEvent> = masters.into_iter().collect();
+        for (ex_graph_id, ex) in exceptions {
+            let Some(master_graph_id) = ex.series_master_id.clone() else {
+                ctx.logger.warn(&format!(
+                    "graph exception event {ex_graph_id} has no seriesMasterId; dropping"
+                ));
+                counts.skipped += 1;
+                continue;
+            };
+            if let Some(master) = master_by_graph_id.get_mut(&master_graph_id) {
+                merge_exception_into(master, &ex);
+            } else if local.contains_key(&master_graph_id) {
+                merge_exception_into_existing(conn, ctx, &master_graph_id, &ex, counts);
+            } else {
+                counts.skipped += 1;
+                ctx.logger.warn(&format!(
+                    "graph exception {ex_graph_id} references missing master {master_graph_id}; orphaned"
+                ));
+            }
+        }
+
+        let pairs: Vec<(String, ConvertedEvent)> = master_by_graph_id.into_iter().collect();
+        insert_events_chunked(conn, ctx, cal.local_id, &pairs, counts)?;
+    }
+
+    if any_failure {
+        ctx.logger.warn(
+            "graph event vanished-cleanup skipped: one or more calendars failed to enumerate; \
+             a clean re-run will reconcile deletions",
+        );
+    } else {
+        delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?;
+    }
+    Ok(())
+}
+
+fn fetch_events(
+    ctx: &GraphCoordinator<'_>,
+    ids: &[String],
+) -> Vec<(String, Result<Value, GraphError>)> {
+    type R = (String, Result<Value, GraphError>);
+    let client = ctx.client.clone();
+    let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone();
+    let body_prefer = match ctx.event_body_format {
+        EventBodyFormat::Text => PREFER_BODY_TEXT,
+        EventBodyFormat::Html => PREFER_BODY_HTML,
+    };
+    let prefer: Vec<String> = vec![PREFER_TIMEZONE_UTC.to_owned(), body_prefer.to_owned()];
+    let pool: Pool<String, R> = Pool::new(ctx.workers, move |id: String| {
+        let url = endpoints.event(&id);
+        let prefer_refs: Vec<&str> = prefer.iter().map(String::as_str).collect();
+        let result = client.get_json_with_prefer(&url, &prefer_refs);
+        (id, result)
+    });
+    for id in ids {
+        pool.submit(id.clone());
+    }
+    let mut out = Vec::with_capacity(ids.len());
+    for _ in 0..ids.len() {
+        if let Ok(r) = pool.results().recv() {
+            out.push(r);
+        }
+    }
+    out
+}
+
+fn merge_exception_into(master: &mut ConvertedEvent, ex: &ConvertedEvent) {
+    let Some(raw_key) = ex.original_start.as_deref() else {
+        return;
+    };
+    let master_tz = master
+        .data
+        .get("timeZone")
+        .and_then(Value::as_str)
+        .map(str::to_owned);
+    let key = normalise_override_key(raw_key, master_tz.as_deref());
+    let Value::Object(map) = &mut master.data else {
+        return;
+    };
+    let overrides = map
+        .entry("recurrenceOverrides".to_owned())
+        .or_insert_with(|| Value::Object(serde_json::Map::new()));
+    let Value::Object(overrides) = overrides else {
+        return;
+    };
+    overrides.insert(key, ex.data.clone());
+}
+
+fn merge_exception_into_existing(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    master_graph_id: &str,
+    ex: &ConvertedEvent,
+    counts: &mut TypeCounts,
+) {
+    let local_id = match exchange_graph_ids::local_for_graph_id(
+        conn,
+        ctx.source_id,
+        exchange_graph_ids::CALENDAR_EVENT,
+        master_graph_id,
+    ) {
+        Ok(Some(id)) => id,
+        Ok(None) => {
+            counts.skipped += 1;
+            return;
+        }
+        Err(e) => {
+            counts.failed += 1;
+            ctx.logger.warn(&format!(
+                "graph exception merge: lookup of master {master_graph_id} failed: {e}"
+            ));
+            return;
+        }
+    };
+    let Some(raw_key) = ex.original_start.as_deref() else {
+        counts.skipped += 1;
+        return;
+    };
+    if let Err(e) = merge_persisted_master(conn, local_id, raw_key, &ex.data) {
+        counts.failed += 1;
+        ctx.logger.warn(&format!(
+            "graph exception merge into stored master {master_graph_id} failed: {e}"
+        ));
+    } else {
+        counts.fetched += 1;
+    }
+}
+
+fn merge_persisted_master(
+    conn: &Connection,
+    local_id: i64,
+    raw_key: &str,
+    ex_data: &Value,
+) -> Result<(), String> {
+    let tx = conn.unchecked_transaction().map_err(|e| e.to_string())?;
+    let row: String = tx
+        .query_row(
+            "SELECT data FROM calendar_events WHERE id = ?1",
+            params![local_id],
+            |row| row.get(0),
+        )
+        .map_err(|e| e.to_string())?;
+    let mut data: Value = serde_json::from_str(&row).map_err(|e| e.to_string())?;
+    let master_tz = data
+        .get("timeZone")
+        .and_then(Value::as_str)
+        .map(str::to_owned);
+    let key = normalise_override_key(raw_key, master_tz.as_deref());
+    if let Value::Object(map) = &mut data {
+        let entry = map
+            .entry("recurrenceOverrides".to_owned())
+            .or_insert_with(|| Value::Object(serde_json::Map::new()));
+        if let Value::Object(overrides) = entry {
+            overrides.insert(key, ex_data.clone());
+        }
+    }
+    tx.execute(
+        "UPDATE calendar_events SET data = ?1 WHERE id = ?2",
+        params![data.to_string(), local_id],
+    )
+    .map_err(|e| e.to_string())?;
+    tx.commit().map_err(|e| e.to_string())?;
+    Ok(())
+}
+
+pub fn normalise_override_key(raw: &str, master_tz: Option<&str>) -> String {
+    if let Some(tz_name) = master_tz
+        && let Some(local) = utc_offset_to_local(raw, tz_name)
+    {
+        return local;
+    }
+    strip_offset_and_fractional(raw)
+}
+
+fn utc_offset_to_local(raw: &str, tz_name: &str) -> Option<String> {
+    use chrono::{DateTime, NaiveDateTime, TimeZone};
+    use chrono_tz::Tz;
+    let tz: Tz = tz_name.parse().ok()?;
+    let parsed: DateTime<chrono::FixedOffset> = if let Ok(dt) = DateTime::parse_from_rfc3339(raw) {
+        dt
+    } else if let Ok(naive) = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S%.f") {
+        return Some(format_local(naive));
+    } else if let Ok(naive) = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S") {
+        return Some(format_local(naive));
+    } else {
+        return None;
+    };
+    let local = tz.from_utc_datetime(&parsed.naive_utc()).naive_local();
+    Some(format_local(local))
+}
+
+fn format_local(dt: chrono::NaiveDateTime) -> String {
+    dt.format("%Y-%m-%dT%H:%M:%S").to_string()
+}
+
+fn strip_offset_and_fractional(raw: &str) -> String {
+    let no_offset = if let Some(idx) = raw.find('Z') {
+        &raw[..idx]
+    } else if let Some(idx) = raw.find('+') {
+        &raw[..idx]
+    } else if let Some(idx) = raw.rfind('-')
+        && idx > 10
+    {
+        &raw[..idx]
+    } else {
+        raw
+    };
+    let trimmed = no_offset.split('.').next().unwrap_or(no_offset);
+    if trimmed.contains('T') {
+        trimmed.to_owned()
+    } else {
+        format!("{trimmed}T00:00:00")
+    }
+}
+
+fn insert_events_chunked(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    calendar_local_id: i64,
+    pairs: &[(String, ConvertedEvent)],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for chunk in pairs.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for (graph_id, event) in chunk {
+            insert_event_in_tx(&tx, ctx, calendar_local_id, graph_id, event, counts)?;
+        }
+        tx.commit()?;
+    }
+    Ok(())
+}
+
+fn insert_event_in_tx(
+    tx: &Transaction<'_>,
+    ctx: &GraphCoordinator<'_>,
+    calendar_local_id: i64,
+    graph_id: &str,
+    event: &ConvertedEvent,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let calendar_ids = json!([calendar_local_id]).to_string();
+    let data = event.data.to_string();
+    tx.execute(
+        "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type)
+         VALUES (?1, ?2, ?3, ?4, 'Event')",
+        params![
+            calendar_ids,
+            event.is_draft as i64,
+            event.use_default_alerts as i64,
+            data,
+        ],
+    )?;
+    let new_id = tx.last_insert_rowid();
+    exchange_graph_ids::insert(
+        tx,
+        ctx.source_id,
+        exchange_graph_ids::CALENDAR_EVENT,
+        graph_id,
+        new_id,
+    )?;
+    counts.created += 1;
+    Ok(())
+}
+
+fn delete_vanished(
+    conn: &mut Connection,
+    source_id: i64,
+    local: &HashMap<String, i64>,
+    server: &std::collections::HashSet<String>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let vanished: Vec<(&String, &i64)> = local
+        .iter()
+        .filter(|(graph_id, _)| !server.contains(graph_id.as_str()))
+        .collect();
+    for chunk in vanished.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for (graph_id, local_id) in chunk {
+            let result = tx.execute(
+                "DELETE FROM calendar_events WHERE id = ?1",
+                params![local_id],
+            );
+            match result {
+                Ok(_) => {
+                    exchange_graph_ids::delete(
+                        &tx,
+                        source_id,
+                        exchange_graph_ids::CALENDAR_EVENT,
+                        graph_id,
+                    )?;
+                    counts.deleted += 1;
+                }
+                Err(_) => {
+                    counts.failed += 1;
+                }
+            }
+        }
+        tx.commit()?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn override_key_utc_converts_to_master_local_time() {
+        let local = normalise_override_key("2026-05-11T15:00:00Z", Some("America/New_York"));
+        assert_eq!(local, "2026-05-11T11:00:00");
+    }
+
+    #[test]
+    fn override_key_no_tz_strips_z_only() {
+        let local = normalise_override_key("2026-05-11T15:00:00Z", None);
+        assert_eq!(local, "2026-05-11T15:00:00");
+    }
+
+    #[test]
+    fn override_key_with_explicit_offset_converts() {
+        let local = normalise_override_key("2026-05-11T15:00:00+00:00", Some("Europe/London"));
+        assert_eq!(local, "2026-05-11T16:00:00");
+    }
+
+    #[test]
+    fn override_key_naive_localdatetime_passes_through() {
+        let local = normalise_override_key("2026-05-11T15:00:00", Some("America/New_York"));
+        assert_eq!(local, "2026-05-11T15:00:00");
+    }
+
+    #[test]
+    fn override_key_unknown_timezone_strips_offset_only() {
+        let local = normalise_override_key("2026-05-11T15:00:00Z", Some("Not/A_Zone"));
+        assert_eq!(local, "2026-05-11T15:00:00");
+    }
+
+    #[test]
+    fn override_key_fractional_seconds_stripped() {
+        let local = normalise_override_key("2026-05-11T15:00:00.1234567Z", None);
+        assert_eq!(local, "2026-05-11T15:00:00");
+    }
+}

src/sync/import_exchange_graph/contacts.rs

@@ -0,0 +1,201 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use rusqlite::{Connection, Transaction, params};
+use serde_json::{Value, json};
+
+use crate::db::exchange_graph_ids;
+use crate::error::Error;
+use crate::exchange_graph::api;
+use crate::exchange_graph::contact_map::{ConvertedContact, convert_contact};
+use crate::exchange_graph::error::GraphError;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+use crate::sync::import_jmap::pool::Pool;
+
+use super::coordinator::{CHUNK_SIZE, GraphCoordinator};
+use super::folders::ContactFolder;
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    books: &[ContactFolder],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let local: HashMap<String, i64> =
+        exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?;
+    let mut server_total: std::collections::HashSet<String> = std::collections::HashSet::new();
+    let mut any_failure = false;
+
+    for book in books {
+        let url = ctx
+            .endpoints
+            .contact_folder_contacts_ids(&book.graph_id, ctx.top);
+        let ids = match api::collect_all_ids(ctx.client, &url, &[]) {
+            Ok(v) => v,
+            Err(e) => {
+                ctx.logger.warn(&format!(
+                    "contact folder {} enumeration failed: {e}",
+                    book.graph_id
+                ));
+                counts.failed += 1;
+                any_failure = true;
+                continue;
+            }
+        };
+        if ctx.logger.enabled(LEVEL_PROGRESS) {
+            eprintln!(
+                "graph contact folder {} contacts: {}",
+                book.graph_id,
+                ids.len()
+            );
+        }
+        for id in &ids {
+            server_total.insert(id.clone());
+        }
+        let new_ids: Vec<String> = ids
+            .into_iter()
+            .filter(|id| !local.contains_key(id))
+            .collect();
+        if new_ids.is_empty() {
+            continue;
+        }
+        let fetched = fetch_contacts(ctx, &new_ids);
+        let mut converted: Vec<(String, ConvertedContact)> = Vec::new();
+        for (graph_id, result) in fetched {
+            match result {
+                Ok(raw) => match convert_contact(&raw) {
+                    Ok(c) => converted.push((graph_id, c)),
+                    Err(e) => {
+                        counts.failed += 1;
+                        ctx.logger
+                            .warn(&format!("graph contact {graph_id} convert failed: {e}"));
+                    }
+                },
+                Err(GraphError::Vanished) => counts.skipped += 1,
+                Err(e) => {
+                    counts.failed += 1;
+                    ctx.logger
+                        .warn(&format!("graph contact {graph_id} fetch failed: {e}"));
+                }
+            }
+        }
+        insert_contacts_chunked(conn, ctx, book.local_id, &converted, counts)?;
+    }
+
+    if any_failure {
+        ctx.logger.warn(
+            "graph contact vanished-cleanup skipped: one or more contact folders failed to enumerate; \
+             a clean re-run will reconcile deletions",
+        );
+    } else {
+        delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?;
+    }
+    Ok(())
+}
+
+fn fetch_contacts(
+    ctx: &GraphCoordinator<'_>,
+    ids: &[String],
+) -> Vec<(String, Result<Value, GraphError>)> {
+    type R = (String, Result<Value, GraphError>);
+    let client = ctx.client.clone();
+    let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone();
+    let pool: Pool<String, R> = Pool::new(ctx.workers, move |id: String| {
+        let url = endpoints.contact(&id);
+        (id, client.get_json_with_prefer(&url, &[]))
+    });
+    for id in ids {
+        pool.submit(id.clone());
+    }
+    let mut out = Vec::with_capacity(ids.len());
+    for _ in 0..ids.len() {
+        if let Ok(r) = pool.results().recv() {
+            out.push(r);
+        }
+    }
+    out
+}
+
+fn insert_contacts_chunked(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    book_local_id: i64,
+    pairs: &[(String, ConvertedContact)],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for chunk in pairs.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for (graph_id, contact) in chunk {
+            insert_contact_in_tx(&tx, ctx, book_local_id, graph_id, contact, counts)?;
+        }
+        tx.commit()?;
+    }
+    Ok(())
+}
+
+fn insert_contact_in_tx(
+    tx: &Transaction<'_>,
+    ctx: &GraphCoordinator<'_>,
+    book_local_id: i64,
+    graph_id: &str,
+    converted: &ConvertedContact,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let book_ids = json!([book_local_id]).to_string();
+    let data = converted.data.to_string();
+    tx.execute(
+        "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)",
+        params![converted.uid, book_ids, data],
+    )?;
+    let new_id = tx.last_insert_rowid();
+    exchange_graph_ids::insert(
+        tx,
+        ctx.source_id,
+        exchange_graph_ids::CONTACT_CARD,
+        graph_id,
+        new_id,
+    )?;
+    counts.created += 1;
+    Ok(())
+}
+
+fn delete_vanished(
+    conn: &mut Connection,
+    source_id: i64,
+    local: &HashMap<String, i64>,
+    server: &std::collections::HashSet<String>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let vanished: Vec<(&String, &i64)> = local
+        .iter()
+        .filter(|(graph_id, _)| !server.contains(graph_id.as_str()))
+        .collect();
+    for chunk in vanished.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for (graph_id, local_id) in chunk {
+            let result = tx.execute("DELETE FROM contact_cards WHERE id = ?1", params![local_id]);
+            match result {
+                Ok(_) => {
+                    exchange_graph_ids::delete(
+                        &tx,
+                        source_id,
+                        exchange_graph_ids::CONTACT_CARD,
+                        graph_id,
+                    )?;
+                    counts.deleted += 1;
+                }
+                Err(_) => {
+                    counts.failed += 1;
+                }
+            }
+        }
+        tx.commit()?;
+    }
+    Ok(())
+}

src/sync/import_exchange_graph/coordinator.rs

@@ -0,0 +1,531 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::Connection;
+use serde_json::Value;
+
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::error::Error;
+use crate::exchange_graph::api::{DEFAULT_API_BASE, Endpoints};
+use crate::exchange_graph::client::GraphClient;
+use crate::exchange_graph::error::GraphError;
+use crate::exchange_graph::oauth::{
+    AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token,
+};
+use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, synthetic_account_id};
+use crate::jmap::http::RetryPolicy;
+use crate::logging::LEVEL_DEFAULT;
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+use crate::types::ObjectType;
+
+#[derive(Debug, Clone)]
+pub enum GraphAuth {
+    PreAcquired {
+        token: String,
+    },
+    DeviceCode {
+        authority: String,
+        client_id: String,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub struct GraphImportConfig {
+    pub auth: GraphAuth,
+    pub api_base: String,
+    pub user_target: Option<String>,
+    pub mailbox_kind: MailboxKind,
+    pub objects: Option<Vec<ObjectType>>,
+    pub event_body_format: EventBodyFormat,
+    pub graph_connections: usize,
+    pub top: usize,
+    pub allow_source_change: bool,
+}
+
+pub const CHUNK_SIZE: usize = 100;
+
+pub struct GraphCoordinator<'a> {
+    pub client: &'a GraphClient,
+    pub endpoints: &'a Endpoints,
+    pub source_id: i64,
+    pub top: usize,
+    pub workers: usize,
+    pub logger: crate::logging::Logger,
+    pub event_body_format: EventBodyFormat,
+}
+
+pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    let mut conn = db::init::open(&common.archive)?;
+
+    let acquired = acquire_with_flow(&config.auth, common.allow_invalid_certs)?;
+    let client = GraphClient::new(
+        acquired.access_token.clone(),
+        RetryPolicy::new(common.max_retries),
+        common.allow_invalid_certs,
+    );
+    client.set_logger(logger);
+
+    let endpoints = resolve_endpoints(&config, &client)?;
+    let principal = resolve_principal(&client, &endpoints, acquired.upn.as_deref())?;
+
+    let session_url = canonical_session_url(&config);
+    let account_id = synthetic_account_id(&principal.id, config.mailbox_kind);
+
+    if !common.dry_run
+        && let Some((url, acc)) =
+            db::sources::conflicting_source(&conn, "exchange_graph", &session_url, &account_id)?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records exchange_graph source ({url}, account {acc}); \
+             pass --allow-source-change to import a different account"
+        )));
+    }
+
+    if matches!(config.mailbox_kind, MailboxKind::Archive) {
+        logger.warn(
+            "--mailbox-kind archive: Microsoft Online Archive holds only mail; \
+             calendar and contact surfaces are skipped for this run",
+        );
+    }
+
+    if common.dry_run {
+        return run_dry(
+            &client,
+            &endpoints,
+            config.mailbox_kind,
+            &logger,
+            config.top,
+        );
+    }
+
+    let source_id = db::sources::upsert_source(
+        &conn,
+        &SourceKey {
+            kind: "exchange_graph".to_owned(),
+            session_url: session_url.clone(),
+            account_id: account_id.clone(),
+        },
+        Some(&principal.user_principal_name),
+        &principal.user_principal_name,
+    )?;
+
+    let _refresher = spawn_token_refresher(
+        &client,
+        &config.auth,
+        &acquired,
+        common.allow_invalid_certs,
+        logger,
+    );
+
+    let mut summary = Summary::default();
+    let mut mailbox_counts = TypeCounts::default();
+    let mut email_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut event_counts = TypeCounts::default();
+    let mut addressbook_counts = TypeCounts::default();
+    let mut contact_counts = TypeCounts::default();
+
+    let ctx = GraphCoordinator {
+        client: &client,
+        endpoints: &endpoints,
+        source_id,
+        top: config.top.clamp(1, 1000),
+        workers: config.graph_connections.clamp(1, 16),
+        logger,
+        event_body_format: config.event_body_format,
+    };
+
+    let want_mail = config
+        .objects
+        .as_ref()
+        .map(|set| {
+            set.iter()
+                .any(|o| matches!(o, ObjectType::Mailbox | ObjectType::Email))
+        })
+        .unwrap_or(true);
+    let want_calendar = config
+        .objects
+        .as_ref()
+        .map(|set| {
+            set.iter()
+                .any(|o| matches!(o, ObjectType::Calendar | ObjectType::CalendarEvent))
+        })
+        .unwrap_or(true)
+        && !matches!(config.mailbox_kind, MailboxKind::Archive);
+    let want_contacts = config
+        .objects
+        .as_ref()
+        .map(|set| {
+            set.iter()
+                .any(|o| matches!(o, ObjectType::AddressBook | ObjectType::ContactCard))
+        })
+        .unwrap_or(true)
+        && !matches!(config.mailbox_kind, MailboxKind::Archive);
+
+    if want_mail {
+        let folders = super::folders::reconcile_mail(
+            &mut conn,
+            &ctx,
+            config.mailbox_kind,
+            &mut mailbox_counts,
+        )?;
+        if logger.enabled(LEVEL_DEFAULT) {
+            eprintln!(
+                "graph mail folders: created={} fetched={} deleted={}",
+                mailbox_counts.created, mailbox_counts.fetched, mailbox_counts.deleted
+            );
+        }
+        super::messages::reconcile_all(&mut conn, &ctx, &folders, &mut email_counts)?;
+    }
+
+    if want_calendar {
+        let calendars = super::folders::reconcile_calendars(&mut conn, &ctx, &mut calendar_counts)?;
+        super::calendar::reconcile_all(&mut conn, &ctx, &calendars, &mut event_counts)?;
+    }
+
+    if want_contacts {
+        let books =
+            super::folders::reconcile_address_books(&mut conn, &ctx, &mut addressbook_counts)?;
+        super::contacts::reconcile_all(&mut conn, &ctx, &books, &mut contact_counts)?;
+    }
+
+    summary.per_type.push(("mailbox", mailbox_counts));
+    summary.per_type.push(("email", email_counts));
+    summary.per_type.push(("calendar", calendar_counts));
+    summary.per_type.push(("calendarevent", event_counts));
+    summary.per_type.push(("addressbook", addressbook_counts));
+    summary.per_type.push(("contactcard", contact_counts));
+
+    if !summary.any_failed()
+        && let Err(e) = run_gc(&conn)
+    {
+        logger.warn(&format!("blob GC skipped: {e}"));
+    }
+
+    summary.retries_observed = client.retries_observed();
+    summary.retry_after_sleeps = client.retry_after_sleeps();
+    Ok(summary)
+}
+
+fn run_dry(
+    client: &GraphClient,
+    endpoints: &Endpoints,
+    mailbox_kind: MailboxKind,
+    logger: &crate::logging::Logger,
+    top: usize,
+) -> Result<Summary, Error> {
+    let mut mailbox_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut addressbook_counts = TypeCounts::default();
+    let folders =
+        enumerate_mail_folders(client, endpoints, mailbox_kind, top).map_err(Error::from)?;
+    mailbox_counts.created = folders.len() as u64;
+    if !matches!(mailbox_kind, MailboxKind::Archive) {
+        let calendars =
+            crate::exchange_graph::api::collect_all_values(client, &endpoints.calendars(top), &[])
+                .map_err(Error::from)?;
+        calendar_counts.created = calendars.len() as u64;
+        let books = crate::exchange_graph::api::collect_all_values(
+            client,
+            &endpoints.contact_folders(top),
+            &[],
+        )
+        .map_err(Error::from)?;
+        addressbook_counts.created = books.len() as u64;
+    }
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "dry-run: mailbox={} calendar={} addressbook={}",
+            mailbox_counts.created, calendar_counts.created, addressbook_counts.created
+        );
+    }
+    let mut summary = Summary::default();
+    summary.per_type.push(("mailbox", mailbox_counts));
+    summary.per_type.push(("calendar", calendar_counts));
+    summary.per_type.push(("addressbook", addressbook_counts));
+    Ok(summary)
+}
+
+pub fn enumerate_mail_folders(
+    client: &GraphClient,
+    endpoints: &Endpoints,
+    mailbox_kind: MailboxKind,
+    top: usize,
+) -> Result<Vec<Value>, GraphError> {
+    let mut all = Vec::new();
+    let mut frontier: Vec<String> = Vec::new();
+    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
+    let initial = endpoints.mail_folders_root(mailbox_kind, top);
+    let level = crate::exchange_graph::api::collect_all_values(client, &initial, &[])?;
+    for f in &level {
+        if let Some(id) = f.get("id").and_then(Value::as_str)
+            && seen.insert(id.to_owned())
+        {
+            frontier.push(id.to_owned());
+        }
+    }
+    all.extend(level);
+    while let Some(parent) = frontier.pop() {
+        let url = endpoints.mail_folder_child_folders(&parent, top);
+        let children = crate::exchange_graph::api::collect_all_values(client, &url, &[])?;
+        for c in &children {
+            if let Some(id) = c.get("id").and_then(Value::as_str)
+                && seen.insert(id.to_owned())
+            {
+                frontier.push(id.to_owned());
+            }
+        }
+        all.extend(children);
+    }
+    Ok(all)
+}
+
+fn acquire_with_flow(auth: &GraphAuth, allow_invalid_certs: bool) -> Result<AcquiredToken, Error> {
+    let flow = match auth {
+        GraphAuth::PreAcquired { token } => OAuthFlow::PreAcquired {
+            token: token.clone(),
+        },
+        GraphAuth::DeviceCode {
+            authority,
+            client_id,
+        } => OAuthFlow::DeviceCode {
+            authority: authority.clone(),
+            client_id: client_id.clone(),
+        },
+    };
+    acquire(&flow, allow_invalid_certs).map_err(Error::from)
+}
+
+fn resolve_endpoints(config: &GraphImportConfig, client: &GraphClient) -> Result<Endpoints, Error> {
+    if let Some(target) = config.user_target.as_deref() {
+        let resolved = if looks_like_uuid(target) {
+            target.to_owned()
+        } else {
+            resolve_user_id(client, &config.api_base, target)
+                .map_err(|e| Error::Connection(format!("user resolution: {e}")))?
+        };
+        Ok(Endpoints::for_user(&config.api_base, &resolved))
+    } else {
+        Ok(Endpoints::for_me(&config.api_base))
+    }
+}
+
+fn looks_like_uuid(s: &str) -> bool {
+    s.len() == 36
+        && s.chars().enumerate().all(|(i, c)| match i {
+            8 | 13 | 18 | 23 => c == '-',
+            _ => c.is_ascii_hexdigit(),
+        })
+}
+
+fn resolve_user_id(
+    client: &GraphClient,
+    api_base: &str,
+    upn_or_id: &str,
+) -> Result<String, GraphError> {
+    let url = format!(
+        "{}/users/{}?$select=id,userPrincipalName",
+        api_base.trim_end_matches('/'),
+        upn_or_id
+    );
+    let body = client.get_json(&url)?;
+    let id = body
+        .get("id")
+        .and_then(Value::as_str)
+        .ok_or_else(|| GraphError::Malformed("user resolution missing id".to_owned()))?
+        .to_owned();
+    Ok(id)
+}
+
+fn resolve_principal(
+    client: &GraphClient,
+    endpoints: &Endpoints,
+    fallback_upn: Option<&str>,
+) -> Result<crate::exchange_graph::ResolvedPrincipal, Error> {
+    let url = endpoints.me_select_id_upn();
+    let value = client.get_json(&url).map_err(Error::from)?;
+    let id = value
+        .get("id")
+        .and_then(Value::as_str)
+        .ok_or_else(|| Error::Connection("graph principal has no id".to_owned()))?
+        .to_owned();
+    let upn = value
+        .get("userPrincipalName")
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .or_else(|| fallback_upn.map(str::to_owned))
+        .unwrap_or_default();
+    Ok(crate::exchange_graph::ResolvedPrincipal {
+        id,
+        user_principal_name: upn,
+    })
+}
+
+fn canonical_session_url(config: &GraphImportConfig) -> String {
+    let authority = match &config.auth {
+        GraphAuth::DeviceCode { authority, .. } => authority.clone(),
+        GraphAuth::PreAcquired { .. } => default_authority("common"),
+    };
+    format!(
+        "{}|{}",
+        authority.trim_end_matches('/'),
+        config.api_base.trim_end_matches('/')
+    )
+}
+
+fn spawn_token_refresher(
+    client: &GraphClient,
+    auth: &GraphAuth,
+    initial: &AcquiredToken,
+    allow_invalid_certs: bool,
+    logger: crate::logging::Logger,
+) -> Option<TokenRefresher> {
+    let (authority, client_id) = match auth {
+        GraphAuth::DeviceCode {
+            authority,
+            client_id,
+        } => (authority.clone(), client_id.clone()),
+        GraphAuth::PreAcquired { .. } => return None,
+    };
+    let refresh = initial.refresh_token.clone()?;
+    let mut deadline_unix = initial_deadline_unix(initial)?;
+    let client = client.clone();
+    let mut refresh_token = refresh;
+    let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let shutdown_signal = shutdown.clone();
+    let handle = std::thread::Builder::new()
+        .name("vandelay-graph-token-refresh".to_owned())
+        .spawn(move || {
+            while !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) {
+                let now = unix_now();
+                let refresh_at = deadline_unix.saturating_sub(60);
+                if refresh_at > now {
+                    let wait = std::time::Duration::from_secs(refresh_at - now);
+                    let chunk = std::time::Duration::from_secs(2);
+                    let mut left = wait;
+                    while left > std::time::Duration::ZERO
+                        && !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed)
+                    {
+                        let step = chunk.min(left);
+                        std::thread::sleep(step);
+                        left = left.saturating_sub(step);
+                    }
+                    if shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) {
+                        break;
+                    }
+                }
+                match refresh_access_token(
+                    &authority,
+                    &client_id,
+                    &refresh_token,
+                    allow_invalid_certs,
+                ) {
+                    Ok(tok) => {
+                        client.set_bearer(tok.access_token.clone());
+                        if let Some(new_refresh) = tok.refresh_token {
+                            refresh_token = new_refresh;
+                        }
+                        deadline_unix = unix_now()
+                            + tok.expires_in.unwrap_or_else(|| {
+                                crate::exchange_graph::oauth::decode_jwt_claims(&tok.access_token)
+                                    .and_then(|c| c.exp)
+                                    .and_then(|exp| exp.checked_sub(unix_now()))
+                                    .unwrap_or(50 * 60)
+                            });
+                    }
+                    Err(e) => {
+                        logger.warn(&format!(
+                            "graph token refresh failed: {e}; sleeping 60s before retry"
+                        ));
+                        std::thread::sleep(std::time::Duration::from_secs(60));
+                    }
+                }
+            }
+        })
+        .ok()?;
+    Some(TokenRefresher {
+        shutdown,
+        handle: Some(handle),
+    })
+}
+
+pub struct TokenRefresher {
+    shutdown: std::sync::Arc<std::sync::atomic::AtomicBool>,
+    handle: Option<std::thread::JoinHandle<()>>,
+}
+
+impl Drop for TokenRefresher {
+    fn drop(&mut self) {
+        self.shutdown
+            .store(true, std::sync::atomic::Ordering::Relaxed);
+        if let Some(h) = self.handle.take() {
+            let _ = h.join();
+        }
+    }
+}
+
+fn unix_now() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+fn initial_deadline_unix(initial: &AcquiredToken) -> Option<u64> {
+    if let Some(expires_in) = initial.expires_in {
+        return Some(unix_now() + expires_in);
+    }
+    crate::exchange_graph::oauth::decode_jwt_claims(&initial.access_token).and_then(|c| c.exp)
+}
+
+fn run_gc(conn: &Connection) -> Result<(), Error> {
+    let tx = conn.unchecked_transaction()?;
+    db::blobs::gc_orphan_blobs(&tx)?;
+    tx.commit()?;
+    Ok(())
+}
+
+pub fn default_api_base() -> String {
+    DEFAULT_API_BASE.to_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn canonical_session_url_concatenates_authority_and_base() {
+        let config = GraphImportConfig {
+            auth: GraphAuth::DeviceCode {
+                authority: "https://login.microsoftonline.com/common".to_owned(),
+                client_id: "uuid".to_owned(),
+            },
+            api_base: "https://graph.microsoft.com/v1.0".to_owned(),
+            user_target: None,
+            mailbox_kind: MailboxKind::Primary,
+            objects: None,
+            event_body_format: EventBodyFormat::Text,
+            graph_connections: 4,
+            top: 100,
+            allow_source_change: false,
+        };
+        let url = canonical_session_url(&config);
+        assert_eq!(
+            url,
+            "https://login.microsoftonline.com/common|https://graph.microsoft.com/v1.0"
+        );
+    }
+
+    #[test]
+    fn looks_like_uuid_only_matches_well_formed() {
+        assert!(looks_like_uuid("12345678-1234-1234-1234-123456789abc"));
+        assert!(!looks_like_uuid("[email protected]"));
+        assert!(!looks_like_uuid("12345678123412341234123456789abc"));
+    }
+}

src/sync/import_exchange_graph/folders.rs

@@ -0,0 +1,531 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use rusqlite::{Connection, params};
+use serde_json::Value;
+
+use crate::db::exchange_graph_ids;
+use crate::error::Error;
+use crate::exchange_graph::api;
+use crate::exchange_graph::calendar_map::{graph_calendar_color_to_hex, windows_or_iana_to_iana};
+use crate::exchange_graph::types::MailboxKind;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+
+use super::coordinator::{CHUNK_SIZE, GraphCoordinator, enumerate_mail_folders};
+
+#[derive(Debug, Clone)]
+pub struct MailFolder {
+    pub graph_id: String,
+    pub parent_graph_id: Option<String>,
+    pub display_name: String,
+    pub is_hidden: bool,
+    pub local_id: i64,
+}
+
+#[derive(Debug, Clone)]
+pub struct CalendarFolder {
+    pub graph_id: String,
+    pub local_id: i64,
+}
+
+#[derive(Debug, Clone)]
+pub struct ContactFolder {
+    pub graph_id: String,
+    pub local_id: i64,
+}
+
+pub fn reconcile_mail(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    mailbox_kind: MailboxKind,
+    counts: &mut TypeCounts,
+) -> Result<Vec<MailFolder>, Error> {
+    let server = enumerate_mail_folders(ctx.client, ctx.endpoints, mailbox_kind, ctx.top)
+        .map_err(Error::from)?;
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!("graph mailFolders enumerated: {}", server.len());
+    }
+    let well_known = match mailbox_kind {
+        MailboxKind::Primary => resolve_well_known_roles(ctx),
+        MailboxKind::Archive => HashMap::new(),
+    };
+    let local: HashMap<String, i64> =
+        exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::MAILBOX)?;
+
+    let entries = order_by_parent(server);
+    let mut by_id: HashMap<String, i64> = HashMap::new();
+    let mut out: Vec<MailFolder> = Vec::new();
+    let mut server_ids: Vec<String> = Vec::new();
+
+    for chunk in entries.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for value in chunk {
+            let Some(graph_id) = value.get("id").and_then(Value::as_str) else {
+                continue;
+            };
+            let parent_graph_id = value
+                .get("parentFolderId")
+                .and_then(Value::as_str)
+                .map(str::to_owned);
+            let display_name = value
+                .get("displayName")
+                .and_then(Value::as_str)
+                .unwrap_or("Unnamed")
+                .to_owned();
+            let is_hidden = value
+                .get("isHidden")
+                .and_then(Value::as_bool)
+                .unwrap_or(false);
+            let role = well_known.get(graph_id).copied();
+            let parent_local_id = parent_graph_id
+                .as_deref()
+                .and_then(|p| by_id.get(p))
+                .copied();
+            let existing = local.get(graph_id).copied();
+            server_ids.push(graph_id.to_owned());
+
+            let local_id = if let Some(id) = existing {
+                tx.execute(
+                    "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4
+                     WHERE id = ?5",
+                    params![display_name, parent_local_id, role, !is_hidden as i64, id,],
+                )?;
+                counts.fetched += 1;
+                id
+            } else {
+                tx.execute(
+                    "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+                     VALUES (?1, ?2, ?3, 0, ?4)",
+                    params![display_name, parent_local_id, role, !is_hidden as i64],
+                )?;
+                let new_id = tx.last_insert_rowid();
+                exchange_graph_ids::insert(
+                    &tx,
+                    ctx.source_id,
+                    exchange_graph_ids::MAILBOX,
+                    graph_id,
+                    new_id,
+                )?;
+                counts.created += 1;
+                new_id
+            };
+            by_id.insert(graph_id.to_owned(), local_id);
+            out.push(MailFolder {
+                graph_id: graph_id.to_owned(),
+                parent_graph_id,
+                display_name,
+                is_hidden,
+                local_id,
+            });
+        }
+        tx.commit()?;
+    }
+
+    delete_vanished_mailboxes(
+        conn,
+        ctx.source_id,
+        &local,
+        &server_ids,
+        counts,
+        &ctx.logger,
+    )?;
+
+    Ok(out)
+}
+
+pub fn reconcile_calendars(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    counts: &mut TypeCounts,
+) -> Result<Vec<CalendarFolder>, Error> {
+    let server = api::collect_all_values(ctx.client, &ctx.endpoints.calendars(ctx.top), &[])
+        .map_err(Error::from)?;
+    let mailbox_tz = mailbox_timezone(ctx);
+    let local: HashMap<String, i64> =
+        exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR)?;
+    let mut out: Vec<CalendarFolder> = Vec::new();
+    let mut server_ids: Vec<String> = Vec::new();
+
+    for chunk in server.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for value in chunk {
+            let Some(graph_id) = value.get("id").and_then(Value::as_str) else {
+                continue;
+            };
+            server_ids.push(graph_id.to_owned());
+            let name = value
+                .get("name")
+                .and_then(Value::as_str)
+                .unwrap_or("Calendar")
+                .to_owned();
+            let color = value
+                .get("hexColor")
+                .and_then(Value::as_str)
+                .map(str::to_owned)
+                .or_else(|| {
+                    value
+                        .get("color")
+                        .and_then(Value::as_str)
+                        .and_then(graph_calendar_color_to_hex)
+                        .map(str::to_owned)
+                });
+            let is_default = value
+                .get("isDefaultCalendar")
+                .and_then(Value::as_bool)
+                .unwrap_or(false);
+            let existing = local.get(graph_id).copied();
+            let tz = mailbox_tz.clone();
+            let local_id = if let Some(id) = existing {
+                tx.execute(
+                    "UPDATE calendars SET name = ?1, color = ?2, is_default = ?3, time_zone = ?4
+                     WHERE id = ?5",
+                    params![name, color, is_default as i64, tz, id],
+                )?;
+                counts.fetched += 1;
+                id
+            } else {
+                tx.execute(
+                    "INSERT INTO calendars (name, color, sort_order, is_subscribed, is_visible,
+                                              is_default, include_in_availability, time_zone)
+                     VALUES (?1, ?2, 0, 1, 1, ?3, 'all', ?4)",
+                    params![name, color, is_default as i64, tz],
+                )?;
+                let new_id = tx.last_insert_rowid();
+                exchange_graph_ids::insert(
+                    &tx,
+                    ctx.source_id,
+                    exchange_graph_ids::CALENDAR,
+                    graph_id,
+                    new_id,
+                )?;
+                counts.created += 1;
+                new_id
+            };
+            out.push(CalendarFolder {
+                graph_id: graph_id.to_owned(),
+                local_id,
+            });
+        }
+        tx.commit()?;
+    }
+
+    delete_vanished_flat(
+        conn,
+        ctx.source_id,
+        exchange_graph_ids::CALENDAR,
+        "calendars",
+        &local,
+        &server_ids,
+        counts,
+    )?;
+    Ok(out)
+}
+
+pub fn reconcile_address_books(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    counts: &mut TypeCounts,
+) -> Result<Vec<ContactFolder>, Error> {
+    let mut server =
+        api::collect_all_values(ctx.client, &ctx.endpoints.contact_folders(ctx.top), &[])
+            .map_err(Error::from)?;
+    let mut seen: std::collections::HashSet<String> = server
+        .iter()
+        .filter_map(|f| f.get("id").and_then(Value::as_str))
+        .map(str::to_owned)
+        .collect();
+    let mut frontier: Vec<String> = seen.iter().cloned().collect();
+    while let Some(parent) = frontier.pop() {
+        let url = ctx.endpoints.contact_folder_children(&parent, ctx.top);
+        let children = api::collect_all_values(ctx.client, &url, &[]).map_err(Error::from)?;
+        for c in &children {
+            if let Some(id) = c.get("id").and_then(Value::as_str)
+                && seen.insert(id.to_owned())
+            {
+                frontier.push(id.to_owned());
+            }
+        }
+        server.extend(children);
+    }
+
+    let local: HashMap<String, i64> =
+        exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::ADDRESS_BOOK)?;
+    let mut out: Vec<ContactFolder> = Vec::new();
+    let mut server_ids: Vec<String> = Vec::new();
+    for chunk in server.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for value in chunk {
+            let Some(graph_id) = value.get("id").and_then(Value::as_str) else {
+                continue;
+            };
+            server_ids.push(graph_id.to_owned());
+            let name = value
+                .get("displayName")
+                .and_then(Value::as_str)
+                .unwrap_or("Contacts")
+                .to_owned();
+            let existing = local.get(graph_id).copied();
+            let local_id = if let Some(id) = existing {
+                tx.execute(
+                    "UPDATE address_books SET name = ?1 WHERE id = ?2",
+                    params![name, id],
+                )?;
+                counts.fetched += 1;
+                id
+            } else {
+                tx.execute(
+                    "INSERT INTO address_books (name, sort_order, is_subscribed)
+                     VALUES (?1, 0, 1)",
+                    params![name],
+                )?;
+                let new_id = tx.last_insert_rowid();
+                exchange_graph_ids::insert(
+                    &tx,
+                    ctx.source_id,
+                    exchange_graph_ids::ADDRESS_BOOK,
+                    graph_id,
+                    new_id,
+                )?;
+                counts.created += 1;
+                new_id
+            };
+            out.push(ContactFolder {
+                graph_id: graph_id.to_owned(),
+                local_id,
+            });
+        }
+        tx.commit()?;
+    }
+
+    delete_vanished_flat(
+        conn,
+        ctx.source_id,
+        exchange_graph_ids::ADDRESS_BOOK,
+        "address_books",
+        &local,
+        &server_ids,
+        counts,
+    )?;
+    Ok(out)
+}
+
+fn mailbox_timezone(ctx: &GraphCoordinator<'_>) -> Option<String> {
+    let url = ctx.endpoints.mailbox_settings_timezone();
+    let value = ctx.client.get_json(&url).ok()?;
+    let tz = value.get("timeZone").and_then(Value::as_str)?;
+    windows_or_iana_to_iana(tz)
+}
+
+fn resolve_well_known_roles(ctx: &GraphCoordinator<'_>) -> HashMap<String, &'static str> {
+    let mapping: &[(&str, &str)] = &[
+        ("inbox", "inbox"),
+        ("drafts", "drafts"),
+        ("sentitems", "sent"),
+        ("deleteditems", "trash"),
+        ("junkemail", "junk"),
+        ("archive", "archive"),
+    ];
+    let mut out = HashMap::new();
+    for (short_name, role) in mapping {
+        let url = ctx.endpoints.well_known_folder(short_name);
+        match ctx.client.get_json(&url) {
+            Ok(value) => {
+                if let Some(id) = value.get("id").and_then(Value::as_str) {
+                    out.insert(id.to_owned(), *role);
+                }
+            }
+            Err(_) => continue,
+        }
+    }
+    out
+}
+
+fn order_by_parent(server: Vec<Value>) -> Vec<Value> {
+    let mut parents: HashMap<String, Option<String>> = HashMap::new();
+    for v in &server {
+        if let Some(id) = v.get("id").and_then(Value::as_str) {
+            let parent = v
+                .get("parentFolderId")
+                .and_then(Value::as_str)
+                .map(str::to_owned);
+            parents.insert(id.to_owned(), parent);
+        }
+    }
+    fn depth_of(
+        id: &str,
+        parents: &HashMap<String, Option<String>>,
+        memo: &mut HashMap<String, usize>,
+        seen: &mut std::collections::HashSet<String>,
+    ) -> usize {
+        if let Some(d) = memo.get(id) {
+            return *d;
+        }
+        if !seen.insert(id.to_owned()) {
+            return 0;
+        }
+        let parent = parents.get(id).and_then(|p| p.as_deref());
+        let d = match parent {
+            Some(p) if parents.contains_key(p) => 1 + depth_of(p, parents, memo, seen),
+            _ => 0,
+        };
+        memo.insert(id.to_owned(), d);
+        d
+    }
+    let mut memo: HashMap<String, usize> = HashMap::new();
+    let mut entries: Vec<(usize, Value)> = server
+        .into_iter()
+        .map(|v| {
+            let id = v.get("id").and_then(Value::as_str).unwrap_or("").to_owned();
+            let mut seen = std::collections::HashSet::new();
+            let d = depth_of(&id, &parents, &mut memo, &mut seen);
+            (d, v)
+        })
+        .collect();
+    entries.sort_by_key(|(d, _)| *d);
+    entries.into_iter().map(|(_, v)| v).collect()
+}
+
+fn delete_vanished_flat(
+    conn: &mut Connection,
+    source_id: i64,
+    type_name: &str,
+    table: &str,
+    local: &HashMap<String, i64>,
+    server_ids: &[String],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let server_set: std::collections::HashSet<&str> =
+        server_ids.iter().map(String::as_str).collect();
+    let vanished: Vec<(&String, &i64)> = local
+        .iter()
+        .filter(|(graph_id, _)| !server_set.contains(graph_id.as_str()))
+        .collect();
+    for chunk in vanished.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for (graph_id, local_id) in chunk {
+            let result = tx.execute(
+                &format!("DELETE FROM {table} WHERE id = ?1"),
+                params![local_id],
+            );
+            match result {
+                Ok(_) => {
+                    exchange_graph_ids::delete(&tx, source_id, type_name, graph_id)?;
+                    counts.deleted += 1;
+                }
+                Err(_) => {
+                    counts.failed += 1;
+                }
+            }
+        }
+        tx.commit()?;
+    }
+    Ok(())
+}
+
+fn delete_vanished_mailboxes(
+    conn: &mut Connection,
+    source_id: i64,
+    local: &HashMap<String, i64>,
+    server_ids: &[String],
+    counts: &mut TypeCounts,
+    logger: &crate::logging::Logger,
+) -> Result<(), Error> {
+    let server_set: std::collections::HashSet<&str> =
+        server_ids.iter().map(String::as_str).collect();
+    let mut vanished: Vec<(String, i64)> = local
+        .iter()
+        .filter(|(id, _)| !server_set.contains(id.as_str()))
+        .map(|(id, lid)| (id.clone(), *lid))
+        .collect();
+    let depths = mailbox_depths(conn, &vanished)?;
+    vanished.sort_by_key(|(_, local_id)| std::cmp::Reverse(*depths.get(local_id).unwrap_or(&0)));
+
+    for (graph_id, local_id) in vanished {
+        let tx = conn.unchecked_transaction()?;
+        let result = tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id]);
+        match result {
+            Ok(_) => {
+                exchange_graph_ids::delete(&tx, source_id, exchange_graph_ids::MAILBOX, &graph_id)?;
+                tx.commit()?;
+                counts.deleted += 1;
+            }
+            Err(e) => {
+                let _ = tx.rollback();
+                logger.warn(&format!(
+                    "mailbox {graph_id} (local id {local_id}) could not be deleted (live children?): {e}"
+                ));
+                counts.failed += 1;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn mailbox_depths(conn: &Connection, rows: &[(String, i64)]) -> Result<HashMap<i64, usize>, Error> {
+    let mut parents: HashMap<i64, Option<i64>> = HashMap::new();
+    {
+        let mut stmt = conn.prepare("SELECT id, parent_id FROM mailboxes")?;
+        let mut iter = stmt.query([])?;
+        while let Some(row) = iter.next()? {
+            let id: i64 = row.get(0)?;
+            let parent: Option<i64> = row.get(1)?;
+            parents.insert(id, parent);
+        }
+    }
+    let mut depths: HashMap<i64, usize> = HashMap::new();
+    for (_, local_id) in rows {
+        depths.insert(*local_id, depth_of(*local_id, &parents));
+    }
+    Ok(depths)
+}
+
+fn depth_of(id: i64, parents: &HashMap<i64, Option<i64>>) -> usize {
+    let mut depth = 0;
+    let mut cursor = id;
+    let mut seen: std::collections::HashSet<i64> = std::collections::HashSet::new();
+    while seen.insert(cursor) {
+        match parents.get(&cursor).copied().flatten() {
+            Some(p) => {
+                depth += 1;
+                cursor = p;
+            }
+            None => break,
+        }
+    }
+    depth
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn order_by_parent_returns_parents_first() {
+        let input = vec![
+            json!({"id": "C", "parentFolderId": "B"}),
+            json!({"id": "B", "parentFolderId": "A"}),
+            json!({"id": "A"}),
+        ];
+        let out = order_by_parent(input);
+        assert_eq!(out[0]["id"], "A");
+        assert_eq!(out[1]["id"], "B");
+        assert_eq!(out[2]["id"], "C");
+    }
+
+    #[test]
+    fn order_by_parent_tolerates_missing_parent() {
+        let input = vec![
+            json!({"id": "Orphan", "parentFolderId": "Missing"}),
+            json!({"id": "A"}),
+        ];
+        let out = order_by_parent(input);
+        assert_eq!(out.len(), 2);
+    }
+}

src/sync/import_exchange_graph/messages.rs

@@ -0,0 +1,202 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::{Connection, Transaction, params};
+use serde_json::json;
+
+use crate::db::{blobs, exchange_graph_ids};
+use crate::error::Error;
+use crate::exchange_graph::api;
+use crate::exchange_graph::client::Accept;
+use crate::exchange_graph::error::GraphError;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+use crate::sync::emailmeta::email_meta_from_blob;
+use crate::sync::import_jmap::pool::Pool;
+use crate::sync::keys::index_to_json;
+
+use super::coordinator::{CHUNK_SIZE, GraphCoordinator};
+use super::folders::MailFolder;
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    folders: &[MailFolder],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let local: std::collections::HashMap<String, i64> =
+        exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?;
+    let mut server_total: std::collections::HashSet<String> = std::collections::HashSet::new();
+    let mut any_failure = false;
+
+    for folder in folders {
+        let url = ctx.endpoints.folder_messages_ids(&folder.graph_id, ctx.top);
+        let ids = match api::collect_all_ids(ctx.client, &url, &[]) {
+            Ok(v) => v,
+            Err(e) => {
+                ctx.logger.warn(&format!(
+                    "folder {} message enumeration failed: {e}",
+                    folder.graph_id
+                ));
+                counts.failed += 1;
+                any_failure = true;
+                continue;
+            }
+        };
+        if ctx.logger.enabled(LEVEL_PROGRESS) {
+            eprintln!(
+                "graph folder {} enumerated {} messages",
+                folder.graph_id,
+                ids.len()
+            );
+        }
+        for id in &ids {
+            server_total.insert(id.clone());
+        }
+        let new_ids: Vec<String> = ids
+            .into_iter()
+            .filter(|id| !local.contains_key(id))
+            .collect();
+        if new_ids.is_empty() {
+            continue;
+        }
+        fetch_and_insert(conn, ctx, folder, &new_ids, counts)?;
+    }
+
+    if any_failure {
+        ctx.logger.warn(
+            "graph message vanished-cleanup skipped: one or more folders failed to enumerate; \
+             a clean re-run will reconcile deletions",
+        );
+    } else {
+        delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?;
+    }
+    Ok(())
+}
+
+fn fetch_and_insert(
+    conn: &mut Connection,
+    ctx: &GraphCoordinator<'_>,
+    folder: &MailFolder,
+    ids: &[String],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let client = ctx.client.clone();
+    let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone();
+    type FetchResult = (String, Result<Vec<u8>, GraphError>);
+    let pool: Pool<String, FetchResult> = Pool::new(ctx.workers, move |id: String| {
+        let url = endpoints.message_mime(&id);
+        match client.get_with_prefer(&url, Accept::Text, &[]) {
+            Ok(resp) => (id, Ok(resp.body)),
+            Err(e) => (id, Err(e)),
+        }
+    });
+    for id in ids {
+        pool.submit(id.clone());
+    }
+    let mut tx_opt: Option<Transaction<'_>> = None;
+    let mut in_batch: usize = 0;
+    for _ in 0..ids.len() {
+        let Ok((graph_id, result)) = pool.results().recv() else {
+            break;
+        };
+        match result {
+            Ok(bytes) => {
+                if tx_opt.is_none() {
+                    tx_opt = Some(conn.unchecked_transaction()?);
+                }
+                let tx = tx_opt.as_mut().expect("tx is Some");
+                apply_message_in_tx(tx, ctx, folder, &graph_id, &bytes, counts)?;
+                in_batch += 1;
+                if in_batch >= CHUNK_SIZE {
+                    if let Some(t) = tx_opt.take() {
+                        t.commit()?;
+                    }
+                    in_batch = 0;
+                }
+            }
+            Err(GraphError::Vanished) => {
+                counts.skipped += 1;
+            }
+            Err(e) => {
+                counts.failed += 1;
+                ctx.logger
+                    .warn(&format!("graph message {graph_id} fetch failed: {e}"));
+            }
+        }
+    }
+    if let Some(t) = tx_opt.take() {
+        t.commit()?;
+    }
+    Ok(())
+}
+
+fn apply_message_in_tx(
+    tx: &Transaction<'_>,
+    ctx: &GraphCoordinator<'_>,
+    folder: &MailFolder,
+    graph_id: &str,
+    bytes: &[u8],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let (idx, date_header) = email_meta_from_blob(bytes);
+    let message_match = index_to_json(&idx);
+    let received_at = date_header.unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned());
+    let mailbox_ids = json!([folder.local_id]).to_string();
+    let keywords = "[]".to_owned();
+
+    let blob_id = blobs::intern_blob(tx, bytes)?;
+    tx.execute(
+        "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match)
+         VALUES (?1, ?2, ?3, ?4, ?5)",
+        params![blob_id, received_at, mailbox_ids, keywords, message_match],
+    )?;
+    let new_id = tx.last_insert_rowid();
+    exchange_graph_ids::insert(
+        tx,
+        ctx.source_id,
+        exchange_graph_ids::EMAIL,
+        graph_id,
+        new_id,
+    )?;
+    counts.created += 1;
+    Ok(())
+}
+
+fn delete_vanished(
+    conn: &mut Connection,
+    source_id: i64,
+    local: &std::collections::HashMap<String, i64>,
+    server: &std::collections::HashSet<String>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let vanished: Vec<(&String, &i64)> = local
+        .iter()
+        .filter(|(graph_id, _)| !server.contains(graph_id.as_str()))
+        .collect();
+    for chunk in vanished.chunks(CHUNK_SIZE) {
+        let tx = conn.unchecked_transaction()?;
+        for (graph_id, local_id) in chunk {
+            let result = tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id]);
+            match result {
+                Ok(_) => {
+                    exchange_graph_ids::delete(
+                        &tx,
+                        source_id,
+                        exchange_graph_ids::EMAIL,
+                        graph_id,
+                    )?;
+                    counts.deleted += 1;
+                }
+                Err(_) => {
+                    counts.failed += 1;
+                }
+            }
+        }
+        tx.commit()?;
+    }
+    Ok(())
+}

src/sync/import_imap.rs

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod fetch;
+pub mod folders;
+pub mod internaldate;
+pub mod keywords;
+pub mod messages;
+pub mod pool;
+
+pub mod coordinator;
+
+pub use coordinator::{ImapAuth, ImapImportConfig, run};

src/sync/import_imap/fetch.rs

@@ -0,0 +1,113 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use crate::imap::response::{Untagged, Value};
+
+#[derive(Debug, Default, Clone)]
+pub struct FetchAttrs {
+    pub uid: Option<u32>,
+    pub flags: Vec<String>,
+    pub internaldate: Option<String>,
+    pub size: Option<u64>,
+    pub body: Option<Vec<u8>>,
+}
+
+pub fn extract(u: &Untagged) -> Option<FetchAttrs> {
+    let Untagged::Fetch { items, .. } = u else {
+        return None;
+    };
+    let mut out = FetchAttrs::default();
+    for (name, value) in items {
+        match name.as_str() {
+            "UID" => {
+                if let Some(n) = value.as_number() {
+                    out.uid = Some(n as u32);
+                }
+            }
+            "FLAGS" => {
+                if let Value::List(items) = value {
+                    out.flags = items
+                        .iter()
+                        .filter_map(|v| match v {
+                            Value::Atom(s) | Value::Str(s) => Some(s.clone()),
+                            _ => None,
+                        })
+                        .collect();
+                }
+            }
+            "INTERNALDATE" => {
+                if let Some(s) = value.as_str() {
+                    out.internaldate = Some(s.to_owned());
+                }
+            }
+            "RFC822.SIZE" => {
+                if let Some(n) = value.as_number() {
+                    out.size = Some(n);
+                }
+            }
+            n if n == "BODY[]" || n == "RFC822" => {
+                out.body = match value {
+                    Value::Bytes(b) => Some(b.clone()),
+                    Value::Str(s) => Some(s.clone().into_bytes()),
+                    _ => None,
+                };
+            }
+            _ => {}
+        }
+    }
+    Some(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Cursor;
+
+    fn parse_fetch(input: &[u8]) -> FetchAttrs {
+        let resp = crate::imap::response::parse_response(&mut Cursor::new(input)).unwrap();
+        match resp {
+            crate::imap::response::Response::Untagged(u) => extract(&u).expect("fetch attrs"),
+            _ => panic!("expected untagged"),
+        }
+    }
+
+    #[test]
+    fn extracts_uid_flags_size_internaldate() {
+        let f = parse_fetch(
+            b"* 1 FETCH (UID 42 FLAGS (\\Seen) INTERNALDATE \"01-Jan-2024 12:00:00 +0000\" RFC822.SIZE 4242)\r\n",
+        );
+        assert_eq!(f.uid, Some(42));
+        assert_eq!(f.flags, vec!["\\Seen"]);
+        assert_eq!(
+            f.internaldate.as_deref(),
+            Some("01-Jan-2024 12:00:00 +0000")
+        );
+        assert_eq!(f.size, Some(4242));
+        assert!(f.body.is_none());
+    }
+
+    #[test]
+    fn extracts_body_literal_as_bytes() {
+        let input = b"* 1 FETCH (UID 5 BODY[] {11}\r\nHello world)\r\n";
+        let f = parse_fetch(input);
+        assert_eq!(f.body.as_deref(), Some(&b"Hello world"[..]));
+    }
+
+    #[test]
+    fn ignores_unknown_attrs() {
+        let f = parse_fetch(b"* 1 FETCH (UID 1 X-CUSTOM \"ignored\" MODSEQ (12345))\r\n");
+        assert_eq!(f.uid, Some(1));
+        assert!(f.flags.is_empty());
+        assert!(f.size.is_none());
+    }
+
+    #[test]
+    fn empty_flags_list() {
+        let f = parse_fetch(b"* 1 FETCH (UID 7 FLAGS ())\r\n");
+        assert_eq!(f.uid, Some(7));
+        assert!(f.flags.is_empty());
+    }
+}

src/sync/import_imap/folders.rs

@@ -0,0 +1,504 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+use regex::Regex;
+
+use crate::imap::automap;
+use crate::imap::name::{canonicalise_inbox, decode_mailbox_name_with};
+use crate::imap::response::Untagged;
+
+#[derive(Debug, Clone, Default)]
+pub struct FolderStatus {
+    pub uidvalidity: Option<u64>,
+    pub uidnext: Option<u64>,
+    pub messages: Option<u64>,
+}
+
+#[derive(Debug, Clone)]
+pub struct DiscoveredFolder {
+    pub name: String,
+    pub delimiter: Option<char>,
+    pub attributes: Vec<String>,
+    pub subscribed: bool,
+    pub selectable: bool,
+    pub status: Option<FolderStatus>,
+}
+
+#[derive(Debug, Clone)]
+pub struct ResolvedFolder {
+    pub name: String,
+    pub leaf: String,
+    pub parent_path: Option<String>,
+    pub delimiter: Option<char>,
+    pub role: Option<&'static str>,
+    pub subscribed: bool,
+    pub status: Option<FolderStatus>,
+}
+
+pub struct FolderFilters {
+    pub include: Vec<Regex>,
+    pub exclude: Vec<Regex>,
+    pub exclude_special: Vec<String>,
+    pub explicit: Vec<String>,
+    pub subscribed_only: bool,
+    pub automap_enabled: bool,
+    pub namespace_prefix: String,
+}
+
+pub fn collect_from_list(
+    untagged: &[Untagged],
+    utf8_accept: bool,
+) -> Result<Vec<DiscoveredFolder>, String> {
+    let mut subscribed = HashSet::new();
+    let mut statuses: BTreeMap<String, FolderStatus> = BTreeMap::new();
+    let mut found = BTreeMap::new();
+    for u in untagged {
+        match u {
+            Untagged::Lsub { name, .. } => {
+                let decoded =
+                    decode_mailbox_name_with(name, utf8_accept).map_err(|e| e.to_string())?;
+                subscribed.insert(canonicalise_inbox(&decoded));
+            }
+            Untagged::List {
+                attributes,
+                delimiter,
+                name,
+            } => {
+                let decoded =
+                    decode_mailbox_name_with(name, utf8_accept).map_err(|e| e.to_string())?;
+                let canonical = canonicalise_inbox(&decoded);
+                let attrs_lower: Vec<String> =
+                    attributes.iter().map(|a| a.to_ascii_lowercase()).collect();
+                let selectable = !attrs_lower
+                    .iter()
+                    .any(|a| a == "\\noselect" || a == "\\nonexistent");
+                let extended_subscribed = attrs_lower.iter().any(|a| a == "\\subscribed");
+                if extended_subscribed {
+                    subscribed.insert(canonical.clone());
+                }
+                found.insert(
+                    canonical.clone(),
+                    DiscoveredFolder {
+                        name: canonical,
+                        delimiter: *delimiter,
+                        attributes: attributes.clone(),
+                        subscribed: false,
+                        selectable,
+                        status: None,
+                    },
+                );
+            }
+            Untagged::Status { mailbox, items } => {
+                let decoded =
+                    decode_mailbox_name_with(mailbox, utf8_accept).map_err(|e| e.to_string())?;
+                let canonical = canonicalise_inbox(&decoded);
+                let mut st = FolderStatus::default();
+                if let Some(v) = items.get("UIDVALIDITY") {
+                    st.uidvalidity = Some(*v);
+                }
+                if let Some(v) = items.get("UIDNEXT") {
+                    st.uidnext = Some(*v);
+                }
+                if let Some(v) = items.get("MESSAGES") {
+                    st.messages = Some(*v);
+                }
+                statuses.insert(canonical, st);
+            }
+            _ => {}
+        }
+    }
+    let mut out: Vec<DiscoveredFolder> = found.into_values().collect();
+    for f in &mut out {
+        if subscribed.contains(&f.name) {
+            f.subscribed = true;
+        }
+        if let Some(s) = statuses.remove(&f.name) {
+            f.status = Some(s);
+        }
+    }
+    Ok(out)
+}
+
+pub fn apply_filters(
+    folders: Vec<DiscoveredFolder>,
+    filters: &FolderFilters,
+) -> Vec<ResolvedFolder> {
+    let mut keep: Vec<DiscoveredFolder> = folders
+        .into_iter()
+        .filter(|f| f.selectable)
+        .filter(|f| match_filters(&f.name, filters))
+        .collect();
+    if filters.subscribed_only {
+        keep.retain(|f| f.subscribed);
+    }
+    let mut resolved: Vec<ResolvedFolder> = Vec::with_capacity(keep.len());
+    for f in keep {
+        let role = automap::role_for_folder(
+            &f.name,
+            &f.attributes,
+            &filters.namespace_prefix,
+            filters.automap_enabled,
+        );
+        if let Some(r) = role
+            && filters
+                .exclude_special
+                .iter()
+                .any(|s| s.eq_ignore_ascii_case(r))
+        {
+            continue;
+        }
+        let delim = f.delimiter;
+        let (leaf, parent_path) = split_parent(&f.name, delim);
+        resolved.push(ResolvedFolder {
+            name: f.name,
+            leaf,
+            parent_path,
+            delimiter: delim,
+            role,
+            subscribed: f.subscribed,
+            status: f.status,
+        });
+    }
+    resolved
+}
+
+fn match_filters(name: &str, filters: &FolderFilters) -> bool {
+    if !filters.explicit.is_empty() {
+        return filters.explicit.iter().any(|e| e == name);
+    }
+    if !filters.include.is_empty() && !filters.include.iter().any(|r| r.is_match(name)) {
+        return false;
+    }
+    if filters.exclude.iter().any(|r| r.is_match(name)) {
+        return false;
+    }
+    true
+}
+
+fn split_parent(name: &str, delim: Option<char>) -> (String, Option<String>) {
+    let Some(d) = delim else {
+        return (name.to_owned(), None);
+    };
+    if let Some(idx) = name.rfind(d) {
+        let parent = &name[..idx];
+        let leaf = &name[idx + d.len_utf8()..];
+        (leaf.to_owned(), Some(parent.to_owned()))
+    } else {
+        (name.to_owned(), None)
+    }
+}
+
+pub fn sort_by_depth(resolved: &mut [ResolvedFolder]) {
+    resolved.sort_by(|a, b| {
+        let da = a.parent_path.as_deref().map_or(0, |s| {
+            a.delimiter.map(|d| s.matches(d).count() + 1).unwrap_or(0)
+        });
+        let db = b.parent_path.as_deref().map_or(0, |s| {
+            b.delimiter.map(|d| s.matches(d).count() + 1).unwrap_or(0)
+        });
+        da.cmp(&db).then_with(|| a.name.cmp(&b.name))
+    });
+}
+
+pub fn vanished_folders(local: &HashMap<String, i64>, server: &HashSet<String>) -> Vec<String> {
+    local
+        .keys()
+        .filter(|k| !server.contains(*k))
+        .cloned()
+        .collect()
+}
+
+pub fn vanished_depth_sort(names: &mut [String], delimiter: char) {
+    names.sort_by(|a, b| {
+        let depth_a = a.matches(delimiter).count();
+        let depth_b = b.matches(delimiter).count();
+        depth_b.cmp(&depth_a).then_with(|| a.cmp(b))
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn lst(name: &str, delim: &str, attrs: &[&str]) -> Untagged {
+        Untagged::List {
+            attributes: attrs.iter().map(|s| (*s).to_owned()).collect(),
+            delimiter: delim.chars().next(),
+            name: name.to_owned(),
+        }
+    }
+
+    #[test]
+    fn collect_filters_noselect_and_canonicalises_inbox() {
+        let resps = vec![
+            lst("Inbox", "/", &[]),
+            lst("Sent", "/", &["\\Sent"]),
+            lst("Hidden", "/", &["\\Noselect"]),
+        ];
+        let folders = collect_from_list(&resps, false).unwrap();
+        assert_eq!(folders.len(), 3);
+        let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap();
+        assert!(inbox.selectable);
+        let hidden = folders.iter().find(|f| f.name == "Hidden").unwrap();
+        assert!(!hidden.selectable);
+    }
+
+    #[test]
+    fn collect_marks_subscribed_via_lsub_or_extended() {
+        let resps = vec![
+            lst("Inbox", "/", &[]),
+            lst("Sent", "/", &["\\Subscribed"]),
+            Untagged::Lsub {
+                attributes: vec![],
+                delimiter: Some('/'),
+                name: "Inbox".to_owned(),
+            },
+        ];
+        let folders = collect_from_list(&resps, false).unwrap();
+        let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap();
+        assert!(inbox.subscribed);
+        let sent = folders.iter().find(|f| f.name == "Sent").unwrap();
+        assert!(sent.subscribed);
+    }
+
+    #[test]
+    fn collect_attaches_list_status_data_per_mailbox() {
+        let mut inbox_status: std::collections::BTreeMap<String, u64> =
+            std::collections::BTreeMap::new();
+        inbox_status.insert("UIDVALIDITY".into(), 12345);
+        inbox_status.insert("UIDNEXT".into(), 42);
+        inbox_status.insert("MESSAGES".into(), 7);
+        let resps = vec![
+            lst("INBOX", "/", &[]),
+            lst("Sent", "/", &["\\Sent"]),
+            Untagged::Status {
+                mailbox: "INBOX".to_owned(),
+                items: inbox_status,
+            },
+        ];
+        let folders = collect_from_list(&resps, false).unwrap();
+        let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap();
+        let st = inbox.status.as_ref().expect("INBOX has status");
+        assert_eq!(st.uidvalidity, Some(12345));
+        assert_eq!(st.uidnext, Some(42));
+        assert_eq!(st.messages, Some(7));
+
+        let sent = folders.iter().find(|f| f.name == "Sent").unwrap();
+        assert!(sent.status.is_none());
+    }
+
+    #[test]
+    fn collect_decodes_modified_utf7_names() {
+        let resps = vec![lst("&ZeVnLIqe-", "/", &[])];
+        let folders = collect_from_list(&resps, false).unwrap();
+        assert_eq!(folders[0].name, "日本語");
+    }
+
+    fn filters_default() -> FolderFilters {
+        FolderFilters {
+            include: Vec::new(),
+            exclude: Vec::new(),
+            exclude_special: Vec::new(),
+            explicit: Vec::new(),
+            subscribed_only: false,
+            automap_enabled: true,
+            namespace_prefix: String::new(),
+        }
+    }
+
+    #[test]
+    fn filters_default_keep_everything() {
+        let folders = collect_from_list(
+            &[
+                lst("INBOX", "/", &[]),
+                lst("Sent", "/", &["\\Sent"]),
+                lst("Drafts", "/", &[]),
+            ],
+            false,
+        )
+        .unwrap();
+        let res = apply_filters(folders, &filters_default());
+        assert_eq!(res.len(), 3);
+        let sent = res.iter().find(|f| f.name == "Sent").unwrap();
+        assert_eq!(sent.role, Some("sent"));
+    }
+
+    #[test]
+    fn include_excludes_non_matching() {
+        let folders = collect_from_list(
+            &[
+                lst("INBOX", "/", &[]),
+                lst("Trash", "/", &[]),
+                lst("Sent", "/", &["\\Sent"]),
+            ],
+            false,
+        )
+        .unwrap();
+        let mut f = filters_default();
+        f.include = vec![Regex::new(r"^(INBOX|Sent)$").unwrap()];
+        let res = apply_filters(folders, &f);
+        assert_eq!(res.len(), 2);
+        assert!(res.iter().any(|x| x.name == "INBOX"));
+        assert!(res.iter().any(|x| x.name == "Sent"));
+    }
+
+    #[test]
+    fn exclude_drops_matching() {
+        let folders =
+            collect_from_list(&[lst("INBOX", "/", &[]), lst("Trash", "/", &[])], false).unwrap();
+        let mut f = filters_default();
+        f.exclude = vec![Regex::new(r"^Trash$").unwrap()];
+        let res = apply_filters(folders, &f);
+        assert_eq!(res.len(), 1);
+        assert_eq!(res[0].name, "INBOX");
+    }
+
+    #[test]
+    fn exclude_special_drops_by_role() {
+        let folders = collect_from_list(
+            &[lst("INBOX", "/", &[]), lst("Trash", "/", &["\\Trash"])],
+            false,
+        )
+        .unwrap();
+        let mut f = filters_default();
+        f.exclude_special = vec!["trash".to_owned()];
+        let res = apply_filters(folders, &f);
+        assert_eq!(res.len(), 1);
+        assert_eq!(res[0].name, "INBOX");
+    }
+
+    #[test]
+    fn explicit_folder_list_overrides_include_exclude() {
+        let folders = collect_from_list(
+            &[
+                lst("INBOX", "/", &[]),
+                lst("Sent", "/", &[]),
+                lst("Trash", "/", &[]),
+            ],
+            false,
+        )
+        .unwrap();
+        let mut f = filters_default();
+        f.explicit = vec!["Sent".to_owned()];
+        let res = apply_filters(folders, &f);
+        assert_eq!(res.len(), 1);
+        assert_eq!(res[0].name, "Sent");
+    }
+
+    #[test]
+    fn subscribed_only_filters_unsubscribed() {
+        let mut folders = collect_from_list(
+            &[lst("INBOX", "/", &["\\Subscribed"]), lst("Other", "/", &[])],
+            false,
+        )
+        .unwrap();
+
+        let mut f = filters_default();
+        f.subscribed_only = true;
+
+        for x in &mut folders {
+            if x.name == "INBOX" {
+                x.subscribed = true;
+            }
+        }
+        let res = apply_filters(folders, &f);
+        assert_eq!(res.len(), 1);
+        assert_eq!(res[0].name, "INBOX");
+    }
+
+    #[test]
+    fn split_parent_handles_root_and_nested() {
+        assert_eq!(split_parent("INBOX", Some('/')), ("INBOX".to_owned(), None));
+        assert_eq!(
+            split_parent("Projects/Alpha", Some('/')),
+            ("Alpha".to_owned(), Some("Projects".to_owned()))
+        );
+        assert_eq!(
+            split_parent("Projects/Alpha/Beta", Some('/')),
+            ("Beta".to_owned(), Some("Projects/Alpha".to_owned()))
+        );
+    }
+
+    #[test]
+    fn split_parent_with_no_delimiter_returns_root() {
+        assert_eq!(
+            split_parent("Anything", None),
+            ("Anything".to_owned(), None)
+        );
+    }
+
+    #[test]
+    fn sort_by_depth_places_roots_first() {
+        let mut r = vec![
+            ResolvedFolder {
+                name: "Projects/Alpha".into(),
+                leaf: "Alpha".into(),
+                parent_path: Some("Projects".into()),
+                delimiter: Some('/'),
+                role: None,
+                subscribed: false,
+                status: None,
+            },
+            ResolvedFolder {
+                name: "INBOX".into(),
+                leaf: "INBOX".into(),
+                parent_path: None,
+                delimiter: Some('/'),
+                role: Some("inbox"),
+                subscribed: true,
+                status: None,
+            },
+            ResolvedFolder {
+                name: "Projects".into(),
+                leaf: "Projects".into(),
+                parent_path: None,
+                delimiter: Some('/'),
+                role: None,
+                subscribed: false,
+                status: None,
+            },
+        ];
+        sort_by_depth(&mut r);
+        assert_eq!(r[0].name, "INBOX");
+        assert_eq!(r[1].name, "Projects");
+        assert_eq!(r[2].name, "Projects/Alpha");
+    }
+
+    #[test]
+    fn vanished_folders_is_set_minus() {
+        let mut local = HashMap::new();
+        local.insert("INBOX".to_owned(), 1);
+        local.insert("Old".to_owned(), 2);
+        local.insert("Sent".to_owned(), 3);
+        let mut server = HashSet::new();
+        server.insert("INBOX".to_owned());
+        server.insert("Sent".to_owned());
+        let v = vanished_folders(&local, &server);
+        assert_eq!(v, vec!["Old".to_owned()]);
+    }
+
+    #[test]
+    fn vanished_depth_sort_deepest_first() {
+        let mut names = vec!["A".to_owned(), "A/B/C".to_owned(), "A/B".to_owned()];
+        vanished_depth_sort(&mut names, '/');
+        assert_eq!(names, vec!["A/B/C", "A/B", "A"]);
+    }
+
+    #[test]
+    fn untagged_status_does_not_become_a_folder() {
+        let resps = vec![
+            lst("INBOX", "/", &[]),
+            Untagged::Status {
+                mailbox: "INBOX".to_owned(),
+                items: BTreeMap::new(),
+            },
+        ];
+        let folders = collect_from_list(&resps, false).unwrap();
+        assert_eq!(folders.len(), 1);
+    }
+}

src/sync/import_imap/internaldate.rs

@@ -0,0 +1,152 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use crate::error::Error;
+
+pub fn imap_internaldate_to_rfc3339(s: &str) -> Result<String, Error> {
+    let s = s.trim_start_matches(' ').trim_end();
+    let (day, rest) = s
+        .split_once('-')
+        .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing day in {s:?}")))?;
+    let (mon, rest) = rest
+        .split_once('-')
+        .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing month in {s:?}")))?;
+    let (year, rest) = rest
+        .split_once(' ')
+        .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing year in {s:?}")))?;
+    let (time, zone) = rest
+        .split_once(' ')
+        .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing zone in {s:?}")))?;
+    let (h, ms) = time
+        .split_once(':')
+        .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?;
+    let (m, sec) = ms
+        .split_once(':')
+        .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?;
+    let day: u32 = day
+        .trim()
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE day {day:?}: {e}")))?;
+    let mon_idx = month_to_num(mon)?;
+    let year: i32 = year
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE year {year:?}: {e}")))?;
+    let h: u32 = h
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE hour {h:?}: {e}")))?;
+    let m: u32 = m
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE minute {m:?}: {e}")))?;
+    let sec: u32 = sec
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE second {sec:?}: {e}")))?;
+    let (sign, hh, mm) = parse_zone(zone)?;
+    let mut out = String::with_capacity(32);
+    use std::fmt::Write;
+    let _ = write!(
+        &mut out,
+        "{year:04}-{mon_idx:02}-{day:02}T{h:02}:{m:02}:{sec:02}"
+    );
+    if hh == 0 && mm == 0 {
+        out.push('Z');
+    } else {
+        let _ = write!(&mut out, "{sign}{hh:02}:{mm:02}");
+    }
+    Ok(out)
+}
+
+fn month_to_num(s: &str) -> Result<u32, Error> {
+    let m = match s {
+        "Jan" => 1,
+        "Feb" => 2,
+        "Mar" => 3,
+        "Apr" => 4,
+        "May" => 5,
+        "Jun" => 6,
+        "Jul" => 7,
+        "Aug" => 8,
+        "Sep" => 9,
+        "Oct" => 10,
+        "Nov" => 11,
+        "Dec" => 12,
+        other => return Err(Error::Partial(format!("INTERNALDATE month {other:?}"))),
+    };
+    Ok(m)
+}
+
+fn parse_zone(s: &str) -> Result<(char, u32, u32), Error> {
+    let s = s.trim();
+    if s.len() != 5 {
+        return Err(Error::Partial(format!("INTERNALDATE zone {s:?}")));
+    }
+    let sign = s.as_bytes()[0] as char;
+    if sign != '+' && sign != '-' {
+        return Err(Error::Partial(format!("INTERNALDATE zone sign {sign}")));
+    }
+    let hh: u32 = s[1..3]
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE zone hour: {e}")))?;
+    let mm: u32 = s[3..5]
+        .parse()
+        .map_err(|e| Error::Partial(format!("INTERNALDATE zone min: {e}")))?;
+    Ok((sign, hh, mm))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn utc_zone_becomes_z() {
+        assert_eq!(
+            imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 +0000").unwrap(),
+            "1996-07-17T02:44:25Z"
+        );
+    }
+
+    #[test]
+    fn positive_offset_preserved() {
+        assert_eq!(
+            imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 +0200").unwrap(),
+            "1996-07-17T02:44:25+02:00"
+        );
+    }
+
+    #[test]
+    fn negative_offset_preserved() {
+        assert_eq!(
+            imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 -0700").unwrap(),
+            "1996-07-17T02:44:25-07:00"
+        );
+    }
+
+    #[test]
+    fn single_digit_day_padded() {
+        assert_eq!(
+            imap_internaldate_to_rfc3339("5-Jan-2024 09:05:01 +0000").unwrap(),
+            "2024-01-05T09:05:01Z"
+        );
+    }
+
+    #[test]
+    fn leading_space_single_digit_day() {
+        assert_eq!(
+            imap_internaldate_to_rfc3339(" 5-Jan-2024 09:05:01 +0000").unwrap(),
+            "2024-01-05T09:05:01Z"
+        );
+    }
+
+    #[test]
+    fn invalid_month_errors() {
+        assert!(imap_internaldate_to_rfc3339("1-XYZ-2024 00:00:00 +0000").is_err());
+    }
+
+    #[test]
+    fn malformed_input_errors() {
+        assert!(imap_internaldate_to_rfc3339("garbage").is_err());
+        assert!(imap_internaldate_to_rfc3339("17-Jul-1996").is_err());
+    }
+}

src/sync/import_imap/keywords.rs

@@ -0,0 +1,173 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FlagAction {
+    Keep(&'static str),
+    Drop,
+    Verbatim,
+    Deleted,
+}
+
+pub fn translate_system_flag(flag: &str) -> FlagAction {
+    if !flag.starts_with('\\') {
+        return FlagAction::Verbatim;
+    }
+    match &flag[1..].to_ascii_lowercase()[..] {
+        "seen" => FlagAction::Keep("$seen"),
+        "flagged" => FlagAction::Keep("$flagged"),
+        "answered" => FlagAction::Keep("$answered"),
+        "draft" => FlagAction::Keep("$draft"),
+        "recent" => FlagAction::Drop,
+        "deleted" => FlagAction::Deleted,
+        _ => FlagAction::Verbatim,
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Translation {
+    pub keywords: Vec<String>,
+    pub has_deleted_flag: bool,
+}
+
+pub fn translate_flags(imap_flags: &[String], include_deleted: bool) -> Translation {
+    let mut out: Vec<String> = Vec::with_capacity(imap_flags.len());
+    let mut has_deleted = false;
+    for f in imap_flags {
+        if f.is_empty() {
+            continue;
+        }
+        match translate_system_flag(f) {
+            FlagAction::Keep(k) => {
+                if !out.iter().any(|x| x.eq_ignore_ascii_case(k)) {
+                    out.push(k.to_owned());
+                }
+            }
+            FlagAction::Drop => {}
+            FlagAction::Verbatim => {
+                if !out.iter().any(|x| x.eq_ignore_ascii_case(f)) {
+                    out.push(f.clone());
+                }
+            }
+            FlagAction::Deleted => {
+                has_deleted = true;
+                if include_deleted && !out.iter().any(|x| x.eq_ignore_ascii_case("$deleted")) {
+                    out.push("$deleted".to_owned());
+                }
+            }
+        }
+    }
+    Translation {
+        keywords: out,
+        has_deleted_flag: has_deleted,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn flags(s: &[&str]) -> Vec<String> {
+        s.iter().map(|x| (*x).to_owned()).collect()
+    }
+
+    #[test]
+    fn system_flag_mapping() {
+        assert!(matches!(
+            translate_system_flag("\\Seen"),
+            FlagAction::Keep("$seen")
+        ));
+        assert!(matches!(
+            translate_system_flag("\\Flagged"),
+            FlagAction::Keep("$flagged")
+        ));
+        assert!(matches!(
+            translate_system_flag("\\Answered"),
+            FlagAction::Keep("$answered")
+        ));
+        assert!(matches!(
+            translate_system_flag("\\Draft"),
+            FlagAction::Keep("$draft")
+        ));
+        assert!(matches!(
+            translate_system_flag("\\Recent"),
+            FlagAction::Drop
+        ));
+        assert!(matches!(
+            translate_system_flag("\\Deleted"),
+            FlagAction::Deleted
+        ));
+    }
+
+    #[test]
+    fn unknown_system_flag_passes_verbatim() {
+        assert!(matches!(
+            translate_system_flag("\\Forwarded"),
+            FlagAction::Verbatim
+        ));
+        assert!(matches!(
+            translate_system_flag("\\MDNSent"),
+            FlagAction::Verbatim
+        ));
+    }
+
+    #[test]
+    fn custom_keywords_pass_verbatim_case() {
+        let t = translate_flags(&flags(&["$Junk", "NotJunk", "MyLabel"]), false);
+        assert_eq!(t.keywords, vec!["$Junk", "NotJunk", "MyLabel"]);
+        assert!(!t.has_deleted_flag);
+    }
+
+    #[test]
+    fn translates_seen_and_flagged() {
+        let t = translate_flags(&flags(&["\\Seen", "\\Flagged"]), false);
+        assert_eq!(t.keywords, vec!["$seen", "$flagged"]);
+    }
+
+    #[test]
+    fn drops_recent() {
+        let t = translate_flags(&flags(&["\\Seen", "\\Recent"]), false);
+        assert_eq!(t.keywords, vec!["$seen"]);
+    }
+
+    #[test]
+    fn deleted_flag_without_include_marks_only() {
+        let t = translate_flags(&flags(&["\\Deleted", "\\Seen"]), false);
+        assert!(t.has_deleted_flag);
+        assert_eq!(t.keywords, vec!["$seen"]);
+    }
+
+    #[test]
+    fn deleted_flag_with_include_adds_dollar_deleted() {
+        let t = translate_flags(&flags(&["\\Deleted", "\\Seen"]), true);
+        assert!(t.has_deleted_flag);
+        assert_eq!(t.keywords, vec!["$deleted", "$seen"]);
+    }
+
+    #[test]
+    fn duplicate_keywords_deduped_case_insensitively() {
+        let t = translate_flags(&flags(&["mykey", "MYKEY", "MyKey"]), false);
+        assert_eq!(t.keywords, vec!["mykey"]);
+    }
+
+    #[test]
+    fn empty_flag_strings_ignored() {
+        let t = translate_flags(&flags(&["", "\\Seen", ""]), false);
+        assert_eq!(t.keywords, vec!["$seen"]);
+    }
+
+    #[test]
+    fn order_preserved() {
+        let t = translate_flags(
+            &flags(&["MyLabel", "\\Seen", "$Important", "\\Flagged"]),
+            false,
+        );
+        assert_eq!(
+            t.keywords,
+            vec!["MyLabel", "$seen", "$Important", "$flagged"]
+        );
+    }
+}

src/sync/import_imap/messages.rs

@@ -0,0 +1,158 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashSet;
+
+use crate::imap::client::ImapClient;
+use crate::imap::command;
+use crate::imap::error::ImapError;
+use crate::imap::response::Untagged;
+use crate::imap::retry::is_negotiation_failure;
+
+pub fn select_uids(client: &mut ImapClient) -> Result<Vec<u32>, ImapError> {
+    if client.has_capability("ESEARCH") {
+        match client.run_collect(command::uid_search_esearch_all()) {
+            Ok(resp) => {
+                for u in &resp.untagged {
+                    if let Untagged::Esearch { all, .. } = u {
+                        return Ok(all.clone());
+                    }
+                }
+            }
+            Err(e) if is_negotiation_failure(&e) => {}
+            Err(e) => return Err(e),
+        }
+    }
+    match client.run_collect(command::uid_search_all()) {
+        Ok(resp) => {
+            for u in &resp.untagged {
+                if let Untagged::Search(uids) = u {
+                    return Ok(uids.clone());
+                }
+            }
+            Ok(Vec::new())
+        }
+        Err(e) if is_negotiation_failure(&e) => uids_via_fetch(client),
+        Err(e) => Err(e),
+    }
+}
+
+fn uids_via_fetch(client: &mut ImapClient) -> Result<Vec<u32>, ImapError> {
+    let resp = client.run_collect(command::uid_fetch_all_uids())?;
+    let mut out = Vec::new();
+    for u in resp.untagged {
+        if let Untagged::Fetch { items, .. } = u {
+            for (n, v) in items {
+                if n == "UID"
+                    && let Some(num) = v.as_number()
+                {
+                    out.push(num as u32);
+                }
+            }
+        }
+    }
+    Ok(out)
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct UidDiff {
+    pub new: Vec<u32>,
+    pub vanished: Vec<u32>,
+    pub present: Vec<u32>,
+}
+
+pub fn diff_uids(local: &[u32], server: &[u32]) -> UidDiff {
+    let local_set: HashSet<u32> = local.iter().copied().collect();
+    let server_set: HashSet<u32> = server.iter().copied().collect();
+    let mut new = Vec::new();
+    let mut present = Vec::new();
+    for &uid in server {
+        if local_set.contains(&uid) {
+            present.push(uid);
+        } else {
+            new.push(uid);
+        }
+    }
+    let mut vanished: Vec<u32> = local
+        .iter()
+        .copied()
+        .filter(|u| !server_set.contains(u))
+        .collect();
+    new.sort_unstable();
+    new.dedup();
+    vanished.sort_unstable();
+    vanished.dedup();
+    present.sort_unstable();
+    present.dedup();
+    UidDiff {
+        new,
+        vanished,
+        present,
+    }
+}
+
+pub fn chunks(uids: &[u32], chunk_size: usize) -> Vec<&[u32]> {
+    if chunk_size == 0 {
+        return vec![uids];
+    }
+    uids.chunks(chunk_size).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn diff_basic() {
+        let local = vec![1, 2, 5];
+        let server = vec![2, 3, 5, 7];
+        let d = diff_uids(&local, &server);
+        assert_eq!(d.new, vec![3, 7]);
+        assert_eq!(d.vanished, vec![1]);
+        assert_eq!(d.present, vec![2, 5]);
+    }
+
+    #[test]
+    fn diff_empty_local_is_all_new() {
+        let d = diff_uids(&[], &[1, 2, 3]);
+        assert_eq!(d.new, vec![1, 2, 3]);
+        assert!(d.vanished.is_empty());
+        assert!(d.present.is_empty());
+    }
+
+    #[test]
+    fn diff_empty_server_is_all_vanished() {
+        let d = diff_uids(&[1, 2, 3], &[]);
+        assert_eq!(d.vanished, vec![1, 2, 3]);
+        assert!(d.new.is_empty());
+        assert!(d.present.is_empty());
+    }
+
+    #[test]
+    fn diff_dedups_inputs() {
+        let d = diff_uids(&[1, 1, 2, 2], &[2, 2, 3, 3]);
+        assert_eq!(d.new, vec![3]);
+        assert_eq!(d.vanished, vec![1]);
+        assert_eq!(d.present, vec![2]);
+    }
+
+    #[test]
+    fn chunks_splits_evenly() {
+        let v: Vec<u32> = (1..=10).collect();
+        let c = chunks(&v, 3);
+        assert_eq!(c.len(), 4);
+        assert_eq!(c[0], &[1, 2, 3][..]);
+        assert_eq!(c[3], &[10][..]);
+    }
+
+    #[test]
+    fn chunks_zero_returns_whole() {
+        let v: Vec<u32> = vec![1, 2, 3];
+        let c = chunks(&v, 0);
+        assert_eq!(c.len(), 1);
+        assert_eq!(c[0], &v[..]);
+    }
+}

src/sync/import_imap/pool.rs

@@ -0,0 +1,256 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::sync::Arc;
+use std::thread;
+
+use crossbeam_channel::{Receiver, Sender, unbounded};
+
+use crate::imap::client::{ConnectMode, ImapClient};
+use crate::imap::command;
+use crate::imap::error::ImapError;
+use crate::imap::name::encode_mailbox_name_with;
+use crate::imap::response::Untagged;
+use crate::imap::retry::{BackoffState, Disposition, RetryPolicy, classify};
+use crate::imap::transport::Connector;
+
+use super::coordinator::{Endpoint, ImapAuth, authenticate_client};
+use super::fetch::FetchAttrs;
+
+pub const HARD_CAP: usize = 8;
+
+pub struct FetchJob {
+    pub folder: String,
+    pub uidvalidity: u32,
+    pub uids: Vec<u32>,
+}
+
+pub enum FetchEvent {
+    Item {
+        folder: String,
+        uidvalidity: u32,
+        attrs: FetchAttrs,
+    },
+    ChunkDone {
+        folder: String,
+        uidvalidity: u32,
+        uids_requested: Vec<u32>,
+        outcome: Result<(), ImapError>,
+    },
+}
+
+pub struct WorkerArgs {
+    pub connector: Arc<Connector>,
+    pub endpoint: Arc<Endpoint>,
+    pub mode: ConnectMode,
+    pub auth: ImapAuth,
+    pub compress: bool,
+    pub policy: RetryPolicy,
+    pub backoff: BackoffState,
+}
+
+pub struct WorkerPool {
+    job_tx: Sender<FetchJob>,
+    event_rx: Receiver<FetchEvent>,
+    handles: Vec<thread::JoinHandle<()>>,
+}
+
+impl WorkerPool {
+    pub fn start(args: WorkerArgs, pool_size: usize) -> Result<WorkerPool, ImapError> {
+        let size = pool_size.clamp(1, HARD_CAP);
+        let (job_tx, job_rx) = unbounded::<FetchJob>();
+        let (event_tx, event_rx) = unbounded::<FetchEvent>();
+        let mut handles = Vec::with_capacity(size);
+        let args = Arc::new(args);
+
+        for _ in 0..size {
+            let args = args.clone();
+            let job_rx = job_rx.clone();
+            let event_tx = event_tx.clone();
+            let handle = thread::spawn(move || {
+                worker_loop(args, job_rx, event_tx);
+            });
+            handles.push(handle);
+        }
+
+        Ok(WorkerPool {
+            job_tx,
+            event_rx,
+            handles,
+        })
+    }
+
+    pub fn submit(&self, job: FetchJob) {
+        let _ = self.job_tx.send(job);
+    }
+
+    pub fn recv(&self) -> Result<FetchEvent, crossbeam_channel::RecvError> {
+        self.event_rx.recv()
+    }
+
+    pub fn recv_timeout(
+        &self,
+        timeout: std::time::Duration,
+    ) -> Result<FetchEvent, crossbeam_channel::RecvTimeoutError> {
+        self.event_rx.recv_timeout(timeout)
+    }
+
+    pub fn shutdown(self) {
+        drop(self.job_tx);
+        for h in self.handles {
+            let _ = h.join();
+        }
+    }
+}
+
+fn worker_loop(args: Arc<WorkerArgs>, job_rx: Receiver<FetchJob>, event_tx: Sender<FetchEvent>) {
+    let mut client: Option<ImapClient> = None;
+    let mut current_folder: Option<String> = None;
+    while let Ok(job) = job_rx.recv() {
+        let job_folder = job.folder.clone();
+        let job_uv = job.uidvalidity;
+        let job_uids = job.uids.clone();
+        let event_tx_for_job = event_tx.clone();
+        let outcome = match catch_unwind(AssertUnwindSafe(|| {
+            run_job_with_retry(
+                &args,
+                &mut client,
+                &mut current_folder,
+                &job,
+                &event_tx_for_job,
+            )
+        })) {
+            Ok(r) => r,
+            Err(_) => {
+                client = None;
+                current_folder = None;
+                Err(ImapError::Protocol("worker thread panicked".into()))
+            }
+        };
+        let _ = event_tx.send(FetchEvent::ChunkDone {
+            folder: job_folder,
+            uidvalidity: job_uv,
+            uids_requested: job_uids,
+            outcome,
+        });
+    }
+}
+
+fn run_job_with_retry(
+    args: &WorkerArgs,
+    client_slot: &mut Option<ImapClient>,
+    current_folder: &mut Option<String>,
+    job: &FetchJob,
+    event_tx: &Sender<FetchEvent>,
+) -> Result<(), ImapError> {
+    let mut transient_attempts: u32 = 0;
+    let mut transport_attempts: u32 = 0;
+    loop {
+        if client_slot.is_none() {
+            match connect_and_auth(args) {
+                Ok(c) => {
+                    *client_slot = Some(c);
+                    *current_folder = None;
+                }
+                Err(e) => {
+                    let disp = classify(&e);
+                    if disp == Disposition::TransportDrop
+                        && transport_attempts < args.policy.max_retries
+                    {
+                        transport_attempts += 1;
+                        std::thread::sleep(args.backoff.transport_delay(transport_attempts));
+                        continue;
+                    }
+                    return Err(e);
+                }
+            }
+        }
+        let Some(client) = client_slot.as_mut() else {
+            return Err(ImapError::Protocol(
+                "worker pool: client slot empty after connect".into(),
+            ));
+        };
+        match run_one_job(client, current_folder, job, event_tx) {
+            Ok(()) => {
+                args.backoff.reset();
+                return Ok(());
+            }
+            Err(e) => match classify(&e) {
+                Disposition::TransportDrop => {
+                    *client_slot = None;
+                    *current_folder = None;
+                    if transport_attempts >= args.policy.max_retries {
+                        return Err(e);
+                    }
+                    transport_attempts += 1;
+                    std::thread::sleep(args.backoff.transport_delay(transport_attempts));
+                }
+                Disposition::Transient => {
+                    if transient_attempts >= args.policy.max_retries {
+                        return Err(e);
+                    }
+                    transient_attempts += 1;
+                    std::thread::sleep(args.backoff.next_shared_delay());
+                }
+                _ => return Err(e),
+            },
+        }
+    }
+}
+
+fn connect_and_auth(args: &WorkerArgs) -> Result<ImapClient, ImapError> {
+    let mut client = ImapClient::connect(
+        &args.connector,
+        &args.endpoint.host,
+        args.endpoint.port,
+        args.mode,
+    )?;
+    authenticate_client(&mut client, &args.auth)
+        .map_err(|e| ImapError::AuthFailed(e.to_string()))?;
+    let _ = client.refresh_capabilities();
+    if args.compress && client.has_capability("COMPRESS=DEFLATE") {
+        client.compress_deflate()?;
+    }
+    if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") {
+        let _ = client.enable(&["UTF8=ACCEPT"]);
+    }
+    Ok(client)
+}
+
+fn run_one_job(
+    client: &mut ImapClient,
+    current_folder: &mut Option<String>,
+    job: &FetchJob,
+    event_tx: &Sender<FetchEvent>,
+) -> Result<(), ImapError> {
+    if current_folder.as_deref() != Some(job.folder.as_str()) {
+        let wire = encode_mailbox_name_with(&job.folder, client.utf8_accept());
+        client.run_collect(&command::select(&wire))?;
+        *current_folder = Some(job.folder.clone());
+    }
+    let set = command::format_uid_set(&job.uids, true);
+    let folder = job.folder.clone();
+    let uv = job.uidvalidity;
+    client.run_streamed(
+        &command::uid_fetch(
+            &set,
+            &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODY.PEEK[]"],
+        ),
+        |u| {
+            if let Untagged::Fetch { .. } = &u
+                && let Some(attrs) = super::fetch::extract(&u)
+            {
+                let _ = event_tx.send(FetchEvent::Item {
+                    folder: folder.clone(),
+                    uidvalidity: uv,
+                    attrs,
+                });
+            }
+        },
+    )?;
+    Ok(())
+}

src/sync/import_jmap/diff.rs

@@ -0,0 +1,88 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashSet;
+
+use crate::jmap::wire::JmapId;
+
+pub struct IdSetDiff {
+    pub new: Vec<JmapId>,
+    pub vanished: Vec<JmapId>,
+    pub present: Vec<JmapId>,
+}
+
+pub fn diff(server_ids: &[JmapId], local_ids: &HashSet<String>) -> IdSetDiff {
+    let server_set: HashSet<&str> = server_ids.iter().map(|i| i.0.as_str()).collect();
+    let mut seen: HashSet<&str> = HashSet::new();
+    let mut new = Vec::new();
+    let mut present = Vec::new();
+    for id in server_ids {
+        if !seen.insert(id.0.as_str()) {
+            continue;
+        }
+        if local_ids.contains(&id.0) {
+            present.push(id.clone());
+        } else {
+            new.push(id.clone());
+        }
+    }
+    let mut vanished = Vec::new();
+    for local in local_ids {
+        if !server_set.contains(local.as_str()) {
+            vanished.push(JmapId(local.clone()));
+        }
+    }
+    vanished.sort_by(|a, b| a.0.cmp(&b.0));
+    IdSetDiff {
+        new,
+        vanished,
+        present,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn ids(v: &[&str]) -> Vec<JmapId> {
+        v.iter().map(|s| JmapId((*s).to_owned())).collect()
+    }
+
+    fn locals(v: &[&str]) -> HashSet<String> {
+        v.iter().map(|s| (*s).to_owned()).collect()
+    }
+
+    #[test]
+    fn partitions_new_vanished_present() {
+        let d = diff(&ids(&["a", "b", "c"]), &locals(&["b", "c", "d"]));
+        assert_eq!(d.new, ids(&["a"]));
+        assert_eq!(d.present, ids(&["b", "c"]));
+        assert_eq!(d.vanished, ids(&["d"]));
+    }
+
+    #[test]
+    fn empty_local_means_all_new() {
+        let d = diff(&ids(&["x", "y"]), &locals(&[]));
+        assert_eq!(d.new, ids(&["x", "y"]));
+        assert!(d.vanished.is_empty());
+        assert!(d.present.is_empty());
+    }
+
+    #[test]
+    fn empty_server_means_all_vanished() {
+        let d = diff(&ids(&[]), &locals(&["p", "q"]));
+        assert!(d.new.is_empty());
+        assert_eq!(d.vanished, ids(&["p", "q"]));
+    }
+
+    #[test]
+    fn duplicate_server_ids_are_collapsed() {
+        let d = diff(&ids(&["a", "a", "b"]), &locals(&["a"]));
+        assert_eq!(d.new, ids(&["b"]));
+        assert_eq!(d.present, ids(&["a"]));
+        assert!(d.vanished.is_empty());
+    }
+}

src/sync/import_jmap/pool.rs

@@ -0,0 +1,134 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::thread::JoinHandle;
+
+use crossbeam_channel::{Receiver, Sender, unbounded};
+
+use crate::jmap::session::Limits;
+
+pub fn effective_workers(requested: usize, limits: &Limits, upload: bool) -> usize {
+    let server_cap = if upload {
+        limits.max_concurrent_upload
+    } else {
+        limits.max_concurrent_requests
+    };
+    let cap = server_cap.max(1).min(usize::MAX as u64) as usize;
+    requested.clamp(1, cap)
+}
+
+pub struct Pool<J, R>
+where
+    J: Send + 'static,
+    R: Send + 'static,
+{
+    job_tx: Option<Sender<J>>,
+    result_rx: Receiver<R>,
+    workers: Vec<JoinHandle<()>>,
+}
+
+impl<J, R> Pool<J, R>
+where
+    J: Send + 'static,
+    R: Send + 'static,
+{
+    pub fn new<F>(threads: usize, worker: F) -> Pool<J, R>
+    where
+        F: Fn(J) -> R + Send + Sync + 'static,
+    {
+        let threads = threads.max(1);
+        let (job_tx, job_rx) = unbounded::<J>();
+        let (result_tx, result_rx) = unbounded::<R>();
+        let worker = std::sync::Arc::new(worker);
+        let mut workers = Vec::with_capacity(threads);
+        for _ in 0..threads {
+            let job_rx = job_rx.clone();
+            let result_tx = result_tx.clone();
+            let worker = worker.clone();
+            workers.push(std::thread::spawn(move || {
+                while let Ok(job) = job_rx.recv() {
+                    if result_tx.send(worker(job)).is_err() {
+                        break;
+                    }
+                }
+            }));
+        }
+        Pool {
+            job_tx: Some(job_tx),
+            result_rx,
+            workers,
+        }
+    }
+
+    pub fn submit(&self, job: J) {
+        if let Some(tx) = &self.job_tx {
+            let _ = tx.send(job);
+        }
+    }
+
+    pub fn results(&self) -> &Receiver<R> {
+        &self.result_rx
+    }
+
+    pub fn finish(mut self) -> Vec<R> {
+        drop(self.job_tx.take());
+        for handle in self.workers.drain(..) {
+            let _ = handle.join();
+        }
+        self.result_rx.try_iter().collect()
+    }
+}
+
+impl<J, R> Drop for Pool<J, R>
+where
+    J: Send + 'static,
+    R: Send + 'static,
+{
+    fn drop(&mut self) {
+        drop(self.job_tx.take());
+        for handle in self.workers.drain(..) {
+            let _ = handle.join();
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn limits(req: u64, up: u64) -> Limits {
+        Limits {
+            max_objects_in_get: 500,
+            max_objects_in_set: 500,
+            max_calls_in_request: 16,
+            max_concurrent_requests: req,
+            max_concurrent_upload: up,
+            max_size_request: 10_000_000,
+            max_size_upload: 50_000_000,
+        }
+    }
+
+    #[test]
+    fn worker_count_is_clamped_to_server_limits() {
+        let l = limits(4, 2);
+        assert_eq!(effective_workers(16, &l, false), 4);
+        assert_eq!(effective_workers(16, &l, true), 2);
+        assert_eq!(effective_workers(1, &l, false), 1);
+        assert_eq!(effective_workers(0, &l, false), 1);
+    }
+
+    #[test]
+    fn pool_processes_all_jobs_across_workers() {
+        let pool: Pool<u64, u64> = Pool::new(4, |n| n * n);
+        for n in 0..100 {
+            pool.submit(n);
+        }
+        let mut got = pool.finish();
+        got.sort_unstable();
+        let expected: Vec<u64> = (0..100).map(|n| n * n).collect();
+        assert_eq!(got, expected);
+    }
+}

src/sync/import_jmap/tree.rs

@@ -0,0 +1,131 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+
+pub struct TreeOrder {
+    pub order: Vec<usize>,
+    pub orphans: Vec<usize>,
+    pub cycle_roots: Vec<usize>,
+}
+
+pub fn topo_order(
+    items: &[(String, Option<String>)],
+    existing_parents: &HashSet<String>,
+) -> TreeOrder {
+    let index: HashMap<&str, usize> = items
+        .iter()
+        .enumerate()
+        .map(|(i, (id, _))| (id.as_str(), i))
+        .collect();
+
+    let mut order = Vec::with_capacity(items.len());
+    let mut orphans = Vec::new();
+    let mut cycle_roots = Vec::new();
+    let mut placed: HashSet<usize> = HashSet::new();
+
+    loop {
+        let mut progressed = false;
+        for (i, (_, parent)) in items.iter().enumerate() {
+            if placed.contains(&i) {
+                continue;
+            }
+            let ready = match parent {
+                None => true,
+                Some(p) => match index.get(p.as_str()) {
+                    Some(&pi) => placed.contains(&pi),
+                    None => {
+                        if !existing_parents.contains(p) {
+                            orphans.push(i);
+                        }
+                        true
+                    }
+                },
+            };
+            if ready {
+                order.push(i);
+                placed.insert(i);
+                progressed = true;
+            }
+        }
+        if placed.len() == items.len() {
+            break;
+        }
+        if !progressed {
+            let mut remaining: Vec<usize> =
+                (0..items.len()).filter(|i| !placed.contains(i)).collect();
+            remaining.sort_by(|&a, &b| items[a].0.cmp(&items[b].0));
+            let root = remaining[0];
+            cycle_roots.push(root);
+            order.push(root);
+            placed.insert(root);
+        }
+    }
+
+    TreeOrder {
+        order,
+        orphans,
+        cycle_roots,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn item(id: &str, parent: Option<&str>) -> (String, Option<String>) {
+        (id.to_owned(), parent.map(str::to_owned))
+    }
+
+    fn empty() -> HashSet<String> {
+        HashSet::new()
+    }
+
+    fn pos(order: &[usize], items: &[(String, Option<String>)], id: &str) -> usize {
+        order
+            .iter()
+            .position(|&i| items[i].0 == id)
+            .expect("id in order")
+    }
+
+    #[test]
+    fn parents_precede_children() {
+        let items = vec![item("c", Some("b")), item("b", Some("a")), item("a", None)];
+        let t = topo_order(&items, &empty());
+        assert_eq!(t.order.len(), 3);
+        assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b"));
+        assert!(pos(&t.order, &items, "b") < pos(&t.order, &items, "c"));
+        assert!(t.orphans.is_empty());
+        assert!(t.cycle_roots.is_empty());
+    }
+
+    #[test]
+    fn parent_not_fetched_and_not_existing_is_orphan() {
+        let items = vec![item("x", Some("missing"))];
+        let t = topo_order(&items, &empty());
+        assert_eq!(t.order, vec![0]);
+        assert_eq!(t.orphans, vec![0]);
+    }
+
+    #[test]
+    fn parent_already_in_db_is_not_orphan() {
+        let items = vec![item("x", Some("p"))];
+        let existing: HashSet<String> = ["p".to_owned()].into_iter().collect();
+        let t = topo_order(&items, &existing);
+        assert_eq!(t.order, vec![0]);
+        assert!(t.orphans.is_empty());
+    }
+
+    #[test]
+    fn cycle_is_broken_by_smallest_id() {
+        let items = vec![item("b", Some("a")), item("a", Some("b"))];
+        let t = topo_order(&items, &empty());
+        assert_eq!(t.order.len(), 2);
+        assert_eq!(t.cycle_roots.len(), 1);
+        assert_eq!(items[t.cycle_roots[0]].0, "a");
+        assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b"));
+    }
+}

src/sync/import_maildir.rs

@@ -0,0 +1,12 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod coordinator;
+pub mod keywords;
+pub mod messages;
+pub mod tree;
+
+pub use coordinator::{MaildirImportConfig, run};

src/sync/import_maildir/coordinator.rs

@@ -0,0 +1,504 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+use std::path::{Path, PathBuf};
+
+use regex::Regex;
+use rusqlite::{Connection, OptionalExtension, params};
+use url::Url;
+
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::error::Error;
+use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+
+use super::messages;
+use super::tree;
+use super::tree::{DiscoverError, FolderFilters, ResolvedFolder};
+
+pub struct MaildirImportConfig {
+    pub maildir: PathBuf,
+    pub include: Vec<Regex>,
+    pub exclude: Vec<Regex>,
+    pub folder: Vec<String>,
+    pub automap: bool,
+    pub include_deleted: bool,
+    pub allow_source_change: bool,
+}
+
+pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    if common.threads > 1 {
+        log_at(
+            logger,
+            LEVEL_PROGRESS,
+            "maildir importer is single-threaded; --threads value will be ignored",
+        );
+    }
+
+    let canonical = std::fs::canonicalize(&config.maildir)
+        .map_err(|e| Error::Usage(format!("--maildir {:?}: {e}", config.maildir)))?;
+
+    warn_if_looks_like_subfolder(&canonical, logger);
+
+    let discovered = match tree::discover(&canonical, config.automap) {
+        Ok(v) => v,
+        Err(DiscoverError::NotFound(p)) => {
+            return Err(Error::Usage(format!("maildir path does not exist: {p:?}")));
+        }
+        Err(DiscoverError::NotADirectory(p)) => {
+            return Err(Error::Usage(format!("not a directory: {p:?}")));
+        }
+        Err(DiscoverError::NotAMaildir(p)) => {
+            return Err(Error::Usage(format!(
+                "not a Maildir: {p:?} (missing cur/ subdirectory)"
+            )));
+        }
+        Err(DiscoverError::NotMaildirPlus(name)) => {
+            return Err(Error::Usage(format!(
+                "only Maildir++ layout is supported; found non-prefixed subfolder {name:?} \
+                 (subfolders must be named with a leading '.', e.g. '.{name}')"
+            )));
+        }
+        Err(DiscoverError::Io(p, e)) => {
+            return Err(Error::Usage(format!("walking {p:?}: {e}")));
+        }
+    };
+
+    let include_deleted = config.include_deleted;
+    let filters = FolderFilters {
+        include: config.include,
+        exclude: config.exclude,
+        explicit: config.folder,
+    };
+    if !filters.explicit.is_empty() && (!filters.include.is_empty() || !filters.exclude.is_empty())
+    {
+        return Err(Error::Usage(
+            "--folder is mutually exclusive with --include/--exclude".to_owned(),
+        ));
+    }
+    let mut resolved = tree::apply_filters(discovered, &filters);
+    tree::restore_ephemeral_parents(&mut resolved, &canonical);
+    let run_flags = RunFlags { include_deleted };
+
+    let mut conn = db::init::open(&common.archive)?;
+    let session_url = file_url_for(&canonical)?;
+    let account_id = canonical.to_string_lossy().into_owned();
+    let account_name = canonical
+        .file_name()
+        .and_then(|n| n.to_str())
+        .map(str::to_owned);
+
+    if let Some((existing_url, existing_account)) =
+        db::sources::conflicting_source(&conn, "maildir", &session_url, &account_id)?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records maildir source {existing_url} / {existing_account}; \
+             re-run with --allow-source-change or use a fresh archive"
+        )));
+    }
+    let source_key = SourceKey {
+        kind: "maildir".to_owned(),
+        session_url: session_url.clone(),
+        account_id: account_id.clone(),
+    };
+    let source_id = if common.dry_run {
+        db::sources::find_source(&conn, &source_key)?.unwrap_or(-1)
+    } else {
+        db::sources::upsert_source(&conn, &source_key, account_name.as_deref(), "")?
+    };
+
+    log_at(
+        logger,
+        LEVEL_PROGRESS,
+        &format!("maildir source {session_url}"),
+    );
+
+    if common.dry_run {
+        let summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?;
+        return Ok(summary);
+    }
+
+    let mut mailbox_counts = TypeCounts::default();
+    let mut email_counts = TypeCounts::default();
+
+    let local_mailboxes = db::maildir_ids::mailbox_folders(&conn, source_id)?;
+    let mailbox_local_ids = upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?;
+
+    let server_names: HashSet<&str> = resolved.iter().map(|f| f.name.as_str()).collect();
+    let mut vanished: Vec<String> = local_mailboxes
+        .keys()
+        .filter(|n| !server_names.contains(n.as_str()))
+        .cloned()
+        .collect();
+    tree::vanished_depth_sort(&mut vanished);
+    if !vanished.is_empty() {
+        delete_vanished_folders(
+            &mut conn,
+            source_id,
+            &vanished,
+            &mut mailbox_counts,
+            &mut email_counts,
+            logger,
+        )?;
+    }
+
+    for folder in &resolved {
+        if folder.ephemeral {
+            continue;
+        }
+        let mailbox_local = match mailbox_local_ids.get(&folder.name).copied() {
+            Some(id) => id,
+            None => {
+                log_at(
+                    logger,
+                    LEVEL_DEFAULT,
+                    &format!("folder {:?}: missing from mailbox id map", folder.name),
+                );
+                email_counts.failed += 1;
+                continue;
+            }
+        };
+        if let Err(e) = reconcile_folder(
+            &mut conn,
+            source_id,
+            folder,
+            mailbox_local,
+            &run_flags,
+            &mut email_counts,
+            logger,
+        ) {
+            log_at(
+                logger,
+                LEVEL_DEFAULT,
+                &format!("folder {:?}: {e}", folder.name),
+            );
+            email_counts.failed += 1;
+        }
+    }
+
+    if email_counts.failed == 0 && mailbox_counts.failed == 0 {
+        let tx = conn.unchecked_transaction()?;
+        db::blobs::gc_orphan_blobs(&tx)?;
+        tx.commit()?;
+    }
+
+    log_at(
+        logger,
+        LEVEL_DEFAULT,
+        &format!(
+            "maildir: mailbox created={} deleted={} skipped={} failed={}; \
+             email created={} fetched={} deleted={} skipped={} failed={}",
+            mailbox_counts.created,
+            mailbox_counts.deleted,
+            mailbox_counts.skipped,
+            mailbox_counts.failed,
+            email_counts.created,
+            email_counts.fetched,
+            email_counts.deleted,
+            email_counts.skipped,
+            email_counts.failed
+        ),
+    );
+
+    Ok(Summary {
+        per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
+        retries_observed: 0,
+        retry_after_sleeps: 0,
+    })
+}
+
+struct RunFlags {
+    include_deleted: bool,
+}
+
+fn file_url_for(p: &Path) -> Result<String, Error> {
+    Url::from_file_path(p).map(|u| u.to_string()).map_err(|_| {
+        Error::Usage(format!(
+            "cannot encode maildir path as a file:// URL: {p:?}"
+        ))
+    })
+}
+
+fn warn_if_looks_like_subfolder(canonical: &Path, logger: Logger) {
+    let Some(basename) = canonical.file_name().and_then(|n| n.to_str()) else {
+        return;
+    };
+    if !basename.starts_with('.') {
+        return;
+    }
+    let Some(parent) = canonical.parent() else {
+        return;
+    };
+    if !parent.join("cur").is_dir() {
+        return;
+    }
+    log_at(
+        logger,
+        LEVEL_DEFAULT,
+        &format!(
+            "warning: {canonical:?} looks like a Maildir++ subfolder of {parent:?} \
+             (basename starts with '.' and the parent has its own cur/); proceeding, \
+             but you may have meant to import the parent"
+        ),
+    );
+}
+
+fn upsert_mailboxes(
+    conn: &mut Connection,
+    source_id: i64,
+    folders: &[ResolvedFolder],
+    counts: &mut TypeCounts,
+) -> Result<HashMap<String, i64>, Error> {
+    let tx = conn.transaction()?;
+    let mut local_ids: HashMap<String, i64> = HashMap::new();
+    for folder in folders {
+        let parent_local = match folder.parent_path.as_deref() {
+            Some(p) => local_ids.get(p).copied().or_else(|| {
+                db::maildir_ids::local_for_mailbox(&tx, source_id, p)
+                    .ok()
+                    .flatten()
+            }),
+            None => None,
+        };
+        let is_subscribed: i64 = if folder.ephemeral { 0 } else { 1 };
+        let existing = db::maildir_ids::local_for_mailbox(&tx, source_id, &folder.name)?;
+        let id = if let Some(id) = existing {
+            tx.execute(
+                "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4
+                 WHERE id = ?5",
+                params![folder.leaf, parent_local, folder.role, is_subscribed, id],
+            )?;
+            id
+        } else {
+            tx.execute(
+                "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+                 VALUES (?1, ?2, ?3, 0, ?4)",
+                params![folder.leaf, parent_local, folder.role, is_subscribed],
+            )?;
+            let new_id = tx.last_insert_rowid();
+            db::maildir_ids::insert_mailbox(&tx, source_id, &folder.name, new_id)?;
+            counts.created += 1;
+            counts.fetched += 1;
+            new_id
+        };
+        local_ids.insert(folder.name.clone(), id);
+    }
+    tx.commit()?;
+    Ok(local_ids)
+}
+
+fn delete_vanished_folders(
+    conn: &mut Connection,
+    source_id: i64,
+    folders: &[String],
+    mailbox_counts: &mut TypeCounts,
+    email_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let tx = conn.transaction()?;
+    for name in folders {
+        let local_id = match db::maildir_ids::local_for_mailbox(&tx, source_id, name)? {
+            Some(id) => id,
+            None => continue,
+        };
+        let surviving_child: Option<i64> = tx
+            .query_row(
+                "SELECT id FROM mailboxes WHERE parent_id = ?1 LIMIT 1",
+                params![local_id],
+                |row| row.get(0),
+            )
+            .optional()?;
+        if surviving_child.is_some() {
+            log_at(
+                logger,
+                LEVEL_DEFAULT,
+                &format!(
+                    "folder {name:?}: vanished from disk but still has child mailboxes in the \
+                     archive; skipping delete (parent_id RESTRICT)"
+                ),
+            );
+            mailbox_counts.failed += 1;
+            continue;
+        }
+        let email_ids: Vec<i64> = tx
+            .prepare(
+                "SELECT local_id FROM sync_id_maildir
+                 WHERE source_id = ?1 AND type_name = 'email' AND folder = ?2",
+            )?
+            .query_map(params![source_id, name], |row| row.get(0))?
+            .collect::<Result<Vec<i64>, _>>()?;
+        for eid in &email_ids {
+            tx.execute("DELETE FROM emails WHERE id = ?1", params![eid])?;
+            email_counts.deleted += 1;
+        }
+        db::maildir_ids::delete_all_emails_in_folder(&tx, source_id, name)?;
+        tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id])?;
+        db::maildir_ids::delete_mailbox(&tx, source_id, name)?;
+        mailbox_counts.deleted += 1;
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+fn reconcile_folder(
+    conn: &mut Connection,
+    source_id: i64,
+    folder: &ResolvedFolder,
+    mailbox_local: i64,
+    flags: &RunFlags,
+    counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let listing = match messages::list_folder(&folder.path) {
+        Ok(l) => l,
+        Err(e) => {
+            return Err(Error::Partial(format!(
+                "list {folder:?}: {e}",
+                folder = folder.path
+            )));
+        }
+    };
+    if listing.io_failures > 0 {
+        counts.failed += listing.io_failures;
+    }
+    let local = db::maildir_ids::email_ids_in_folder(conn, source_id, &folder.name)?;
+    let diff = messages::diff(listing.entries, &local);
+
+    log_at(
+        logger,
+        LEVEL_PROGRESS,
+        &format!(
+            "folder {:?}: new={} present={} vanished={}",
+            folder.name,
+            diff.new.len(),
+            diff.present.len(),
+            diff.vanished.len()
+        ),
+    );
+
+    let ctx = messages::InsertContext {
+        source_id,
+        folder: &folder.name,
+        mailbox_local,
+        include_deleted: flags.include_deleted,
+    };
+    messages::apply_folder(conn, ctx, diff, counts, logger)?;
+    Ok(())
+}
+
+fn build_dry_run_summary(
+    conn: &Connection,
+    source_id: i64,
+    folders: &[ResolvedFolder],
+    logger: Logger,
+) -> Result<Summary, Error> {
+    let mut mailbox = TypeCounts::default();
+    let mut email = TypeCounts::default();
+    let local_mailboxes = if source_id < 0 {
+        HashMap::new()
+    } else {
+        db::maildir_ids::mailbox_folders(conn, source_id)?
+    };
+    let server_set: HashSet<&str> = folders.iter().map(|f| f.name.as_str()).collect();
+    let new_folders: Vec<&str> = folders
+        .iter()
+        .map(|f| f.name.as_str())
+        .filter(|n| !local_mailboxes.contains_key(*n))
+        .collect();
+    let vanished_folder_names: Vec<&str> = local_mailboxes
+        .keys()
+        .filter(|n| !server_set.contains(n.as_str()))
+        .map(|s| s.as_str())
+        .collect();
+    mailbox.created += new_folders.len() as u64;
+    mailbox.fetched += new_folders.len() as u64;
+    mailbox.deleted += vanished_folder_names.len() as u64;
+
+    for folder in folders {
+        if folder.ephemeral {
+            continue;
+        }
+        let listing = match messages::list_folder(&folder.path) {
+            Ok(l) => l,
+            Err(e) => {
+                logger.warn(&format!("dry-run list {:?}: {e}", folder.name));
+                email.failed += 1;
+                continue;
+            }
+        };
+        email.failed += listing.io_failures;
+        let disk_ids: HashSet<String> = listing
+            .entries
+            .iter()
+            .map(|e| e.unique_id.clone())
+            .collect();
+        let local = if source_id < 0 {
+            HashMap::new()
+        } else {
+            db::maildir_ids::email_ids_in_folder(conn, source_id, &folder.name)?
+        };
+        let local_keys: HashSet<&String> = local.keys().collect();
+        let new_count = disk_ids.iter().filter(|u| !local.contains_key(*u)).count() as u64;
+        let vanished_count = local_keys
+            .iter()
+            .filter(|k| !disk_ids.contains(k.as_str()))
+            .count() as u64;
+        let present_count = disk_ids.iter().filter(|u| local.contains_key(*u)).count() as u64;
+        email.created += new_count;
+        email.fetched += new_count;
+        email.deleted += vanished_count;
+        email.skipped += present_count;
+    }
+    for name in &vanished_folder_names {
+        if source_id >= 0 {
+            let n = db::maildir_ids::email_ids_in_folder(conn, source_id, name)?.len() as u64;
+            email.deleted += n;
+        }
+    }
+
+    Ok(Summary {
+        per_type: vec![("mailbox", mailbox), ("email", email)],
+        retries_observed: 0,
+        retry_after_sleeps: 0,
+    })
+}
+
+fn log_at(logger: Logger, level: u8, msg: &str) {
+    if logger.enabled(level) {
+        eprintln!("{msg}");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn file_url_for_ascii_path_is_passthrough() {
+        let p = std::path::Path::new("/home/alice/Maildir");
+        let url = file_url_for(p).unwrap();
+        assert_eq!(url, "file:///home/alice/Maildir");
+    }
+
+    #[test]
+    fn file_url_for_percent_encodes_spaces_and_unicode() {
+        let p = std::path::Path::new("/home/alice/My Maildir/Sübfolder");
+        let url = file_url_for(p).unwrap();
+        assert!(url.starts_with("file:///home/alice/My%20Maildir/"));
+        assert!(url.contains("S%C3%BCbfolder"));
+    }
+
+    #[test]
+    fn file_url_for_rejects_relative_paths() {
+        let p = std::path::Path::new("./relative");
+        let err = file_url_for(p).unwrap_err();
+        assert!(matches!(err, Error::Usage(_)));
+    }
+}

src/sync/import_maildir/keywords.rs

@@ -0,0 +1,202 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use mail_parser::mailbox::maildir::Flag;
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct Translation {
+    pub keywords: Vec<String>,
+    pub has_trashed_flag: bool,
+}
+
+pub fn translate_flags(flags: &[Flag], include_deleted: bool) -> Translation {
+    let mut out: Vec<String> = Vec::new();
+    let mut has_trashed = false;
+    for f in flags {
+        match f {
+            Flag::Passed => push_unique(&mut out, "$forwarded"),
+            Flag::Replied => push_unique(&mut out, "$answered"),
+            Flag::Seen => push_unique(&mut out, "$seen"),
+            Flag::Draft => push_unique(&mut out, "$draft"),
+            Flag::Flagged => push_unique(&mut out, "$flagged"),
+            Flag::Trashed => {
+                has_trashed = true;
+                if include_deleted {
+                    push_unique(&mut out, "$deleted");
+                }
+            }
+        }
+    }
+    out.sort();
+    Translation {
+        keywords: out,
+        has_trashed_flag: has_trashed,
+    }
+}
+
+fn push_unique(out: &mut Vec<String>, k: &str) {
+    if !out.iter().any(|x| x == k) {
+        out.push(k.to_owned());
+    }
+}
+
+pub fn flags_from_filename(filename: &str) -> Vec<Flag> {
+    let Some(part) = filename.rsplit_once("2,").map(|(_, s)| s) else {
+        return Vec::new();
+    };
+    let mut out = Vec::new();
+    for &ch in part.as_bytes() {
+        match ch {
+            b'P' => out.push(Flag::Passed),
+            b'R' => out.push(Flag::Replied),
+            b'S' => out.push(Flag::Seen),
+            b'T' => out.push(Flag::Trashed),
+            b'D' => out.push(Flag::Draft),
+            b'F' => out.push(Flag::Flagged),
+            other => {
+                if !other.is_ascii_alphanumeric() {
+                    break;
+                }
+            }
+        }
+    }
+    out
+}
+
+pub fn unique_id_from_filename(filename: &str) -> &str {
+    match filename.split_once(':') {
+        Some((unique, _)) => unique,
+        None => filename,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn empty_flags_yields_empty_keywords() {
+        let t = translate_flags(&[], false);
+        assert!(t.keywords.is_empty());
+        assert!(!t.has_trashed_flag);
+    }
+
+    #[test]
+    fn each_flag_maps_to_expected_jmap_keyword() {
+        let t = translate_flags(
+            &[
+                Flag::Passed,
+                Flag::Replied,
+                Flag::Seen,
+                Flag::Draft,
+                Flag::Flagged,
+            ],
+            false,
+        );
+        let mut expected = vec!["$forwarded", "$answered", "$seen", "$draft", "$flagged"];
+        expected.sort();
+        let expected: Vec<String> = expected.into_iter().map(str::to_owned).collect();
+        assert_eq!(t.keywords, expected);
+        assert!(!t.has_trashed_flag);
+    }
+
+    #[test]
+    fn trashed_dropped_by_default_marks_flag() {
+        let t = translate_flags(&[Flag::Trashed, Flag::Seen], false);
+        assert!(t.has_trashed_flag);
+        assert_eq!(t.keywords, vec!["$seen".to_owned()]);
+    }
+
+    #[test]
+    fn trashed_with_include_deleted_adds_dollar_deleted() {
+        let t = translate_flags(&[Flag::Trashed, Flag::Seen], true);
+        assert!(t.has_trashed_flag);
+        let mut expected = vec!["$seen", "$deleted"];
+        expected.sort();
+        let expected: Vec<String> = expected.into_iter().map(str::to_owned).collect();
+        assert_eq!(t.keywords, expected);
+    }
+
+    #[test]
+    fn order_independent_keywords_are_stable() {
+        let a = translate_flags(&[Flag::Seen, Flag::Flagged], false);
+        let b = translate_flags(&[Flag::Flagged, Flag::Seen], false);
+        assert_eq!(a.keywords, b.keywords);
+    }
+
+    #[test]
+    fn duplicate_flags_dedup() {
+        let t = translate_flags(&[Flag::Seen, Flag::Seen, Flag::Seen], false);
+        assert_eq!(t.keywords, vec!["$seen".to_owned()]);
+    }
+
+    #[test]
+    fn unique_id_strips_at_first_colon() {
+        assert_eq!(
+            unique_id_from_filename("1739471123.M001P01234V0.host:2,RS"),
+            "1739471123.M001P01234V0.host"
+        );
+    }
+
+    #[test]
+    fn unique_id_no_colon_is_whole_basename() {
+        assert_eq!(
+            unique_id_from_filename("1739471123.M001P0.host"),
+            "1739471123.M001P0.host"
+        );
+    }
+
+    #[test]
+    fn unique_id_strips_legacy_one_form() {
+        assert_eq!(
+            unique_id_from_filename("1234.M5.host:1,experimental"),
+            "1234.M5.host"
+        );
+    }
+
+    #[test]
+    fn flags_from_filename_recognises_full_set() {
+        let mut flags = flags_from_filename("name:2,DFPRST");
+        flags.sort();
+        let mut want = vec![
+            Flag::Draft,
+            Flag::Flagged,
+            Flag::Passed,
+            Flag::Replied,
+            Flag::Seen,
+            Flag::Trashed,
+        ];
+        want.sort();
+        assert_eq!(flags, want);
+    }
+
+    #[test]
+    fn flags_from_filename_no_info_section_yields_empty() {
+        assert!(flags_from_filename("plain").is_empty());
+        assert!(flags_from_filename("name:").is_empty());
+        assert!(flags_from_filename("name:1,X").is_empty());
+    }
+
+    #[test]
+    fn flags_from_filename_dovecot_extension_metadata_passes_through() {
+
+        let mut flags = flags_from_filename("uid,S=1234,W=1300:2,RS");
+        flags.sort();
+        let mut want = vec![Flag::Replied, Flag::Seen];
+        want.sort();
+        assert_eq!(flags, want);
+    }
+
+    #[test]
+    fn flags_from_filename_unknown_alpha_chars_skipped() {
+
+        let mut flags = flags_from_filename("uid:2,SaT");
+        flags.sort();
+        let mut want = vec![Flag::Seen, Flag::Trashed];
+        want.sort();
+        assert_eq!(flags, want);
+    }
+}

src/sync/import_maildir/messages.rs

@@ -0,0 +1,726 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::time::{Duration, UNIX_EPOCH};
+
+use mail_parser::mailbox::maildir::Flag;
+use rusqlite::{Connection, params};
+use serde_json::Value;
+use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
+
+use crate::db;
+use crate::sync::TypeCounts;
+use crate::sync::emailmeta::email_index_from_blob;
+use crate::sync::keys::index_to_json;
+
+use super::keywords::{Translation, flags_from_filename, translate_flags, unique_id_from_filename};
+
+#[derive(Debug, Clone)]
+pub struct DiskEntry {
+    pub unique_id: String,
+    pub filename: String,
+    pub path: PathBuf,
+    pub flags: Vec<Flag>,
+    pub mtime_unix: u64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DiskListing {
+    pub entries: Vec<DiskEntry>,
+    pub io_failures: u64,
+}
+
+pub fn list_folder(folder_path: &Path) -> std::io::Result<DiskListing> {
+    let mut out = DiskListing::default();
+    let mut seen: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
+    for sub in ["cur", "new"] {
+        let dir = folder_path.join(sub);
+        if !dir.is_dir() {
+            continue;
+        }
+        let entries = std::fs::read_dir(&dir)?;
+        for entry in entries {
+            let entry = match entry {
+                Ok(e) => e,
+                Err(_) => {
+                    out.io_failures += 1;
+                    continue;
+                }
+            };
+            let path = entry.path();
+            let file_type = match entry.file_type() {
+                Ok(t) => t,
+                Err(_) => {
+                    out.io_failures += 1;
+                    continue;
+                }
+            };
+            if !file_type.is_file() {
+                continue;
+            }
+            let Some(raw_name) = path.file_name() else {
+                continue;
+            };
+            let filename: String = raw_name.to_string_lossy().into_owned();
+            if filename.starts_with('.') {
+                continue;
+            }
+            let unique_id = unique_id_from_filename(&filename).to_owned();
+            if unique_id.is_empty() {
+                continue;
+            }
+            if seen.contains_key(&unique_id) {
+
+                continue;
+            }
+            let flags = flags_from_filename(&filename);
+            let mtime_unix = match entry.metadata().and_then(|m| m.modified()).and_then(|t| {
+                t.duration_since(UNIX_EPOCH).map_err(|e| {
+                    std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())
+                })
+            }) {
+                Ok(d) => d.as_secs(),
+                Err(_) => 0,
+            };
+            seen.insert(unique_id.clone(), out.entries.len());
+            out.entries.push(DiskEntry {
+                unique_id,
+                filename,
+                path,
+                flags,
+                mtime_unix,
+            });
+        }
+    }
+    Ok(out)
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct FolderDiff {
+    pub new: Vec<DiskEntry>,
+    pub present: Vec<(DiskEntry, i64)>,
+    pub vanished: Vec<(String, i64)>,
+}
+
+pub fn diff(disk: Vec<DiskEntry>, local: &HashMap<String, i64>) -> FolderDiff {
+    let mut diff = FolderDiff::default();
+    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
+    for entry in disk {
+        match local.get(&entry.unique_id) {
+            None => {
+                seen.insert(entry.unique_id.clone());
+                diff.new.push(entry);
+            }
+            Some(&local_id) => {
+                seen.insert(entry.unique_id.clone());
+                diff.present.push((entry, local_id));
+            }
+        }
+    }
+    for (uid, local_id) in local {
+        if !seen.contains(uid) {
+            diff.vanished.push((uid.clone(), *local_id));
+        }
+    }
+    diff.vanished.sort_by(|a, b| a.0.cmp(&b.0));
+    diff
+}
+
+#[derive(Clone, Copy)]
+pub struct InsertContext<'a> {
+    pub source_id: i64,
+    pub folder: &'a str,
+    pub mailbox_local: i64,
+    pub include_deleted: bool,
+}
+
+pub fn insert_new(
+    tx: &rusqlite::Transaction<'_>,
+    ctx: InsertContext<'_>,
+    entry: &DiskEntry,
+) -> Result<Option<i64>, InsertError> {
+    let bytes = std::fs::read(&entry.path).map_err(InsertError::Io)?;
+    let translation = translate_flags(&entry.flags, ctx.include_deleted);
+    if translation.has_trashed_flag && !ctx.include_deleted {
+        return Ok(None);
+    }
+    let received_at = format_unix_rfc3339(entry.mtime_unix);
+    let blob_id = db::blobs::intern_blob(tx, &bytes)?;
+    let message_match = index_to_json(&email_index_from_blob(&bytes));
+    let mailbox_ids = Value::Array(vec![Value::from(ctx.mailbox_local)]);
+    let keywords = keywords_json(&translation);
+    tx.execute(
+        "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match)
+         VALUES (?1, ?2, ?3, ?4, ?5)",
+        params![
+            blob_id,
+            received_at,
+            mailbox_ids.to_string(),
+            keywords,
+            message_match
+        ],
+    )?;
+    let local_id = tx.last_insert_rowid();
+    db::maildir_ids::insert_email(tx, ctx.source_id, ctx.folder, &entry.unique_id, local_id)?;
+    Ok(Some(local_id))
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PresentOutcome {
+
+    Unchanged,
+
+    KeywordsUpdated,
+
+    DeleteRequested,
+}
+
+pub fn refresh_flags(
+    tx: &rusqlite::Transaction<'_>,
+    local_id: i64,
+    entry: &DiskEntry,
+    stored_keywords: &str,
+    include_deleted: bool,
+) -> Result<PresentOutcome, rusqlite::Error> {
+    let translation = translate_flags(&entry.flags, include_deleted);
+    if translation.has_trashed_flag && !include_deleted {
+        return Ok(PresentOutcome::DeleteRequested);
+    }
+    let expected = keywords_json(&translation);
+    if stored_keywords == expected {
+        return Ok(PresentOutcome::Unchanged);
+    }
+    tx.execute(
+        "UPDATE emails SET keywords = ?1 WHERE id = ?2",
+        params![expected, local_id],
+    )?;
+    Ok(PresentOutcome::KeywordsUpdated)
+}
+
+pub fn load_present_keywords(
+    conn: &Connection,
+    source_id: i64,
+    folder: &str,
+) -> Result<HashMap<i64, String>, rusqlite::Error> {
+    let mut stmt = conn.prepare(
+        "SELECT e.id, e.keywords FROM emails e
+         JOIN sync_id_maildir m ON m.local_id = e.id AND m.type_name = 'email'
+         WHERE m.source_id = ?1 AND m.folder = ?2",
+    )?;
+    let rows = stmt.query_map(params![source_id, folder], |row| {
+        Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
+    })?;
+    let mut out = HashMap::new();
+    for r in rows {
+        let (id, kw) = r?;
+        out.insert(id, kw);
+    }
+    Ok(out)
+}
+
+pub fn delete_vanished(
+    tx: &rusqlite::Transaction<'_>,
+    source_id: i64,
+    folder: &str,
+    unique_id: &str,
+    local_id: i64,
+) -> Result<(), rusqlite::Error> {
+    tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id])?;
+    db::maildir_ids::delete_email(tx, source_id, folder, unique_id)?;
+    Ok(())
+}
+
+const PROGRESS_TICK: u64 = 1000;
+
+pub fn apply_folder(
+    conn: &mut Connection,
+    ctx: InsertContext<'_>,
+    diff: FolderDiff,
+    counts: &mut TypeCounts,
+    logger: crate::logging::Logger,
+) -> Result<(), crate::error::Error> {
+    let stored_keywords =
+        load_present_keywords(conn, ctx.source_id, ctx.folder).unwrap_or_default();
+    let tx = conn.transaction()?;
+    let total_new = diff.new.len() as u64;
+    let mut inserted: u64 = 0;
+    for entry in &diff.new {
+        match insert_new(&tx, ctx, entry) {
+            Ok(Some(_)) => {
+                counts.created += 1;
+                counts.fetched += 1;
+                inserted += 1;
+                if inserted.is_multiple_of(PROGRESS_TICK)
+                    && logger.enabled(crate::logging::LEVEL_PROGRESS)
+                {
+                    eprintln!(
+                        "folder {folder:?}: inserted {inserted}/{total_new}",
+                        folder = ctx.folder
+                    );
+                }
+            }
+            Ok(None) => {
+                counts.skipped += 1;
+            }
+            Err(e) => {
+                logger.warn(&format!(
+                    "maildir {folder:?}/{name}: {e}",
+                    folder = ctx.folder,
+                    name = entry.filename
+                ));
+                counts.failed += 1;
+            }
+        }
+    }
+    for (entry, local_id) in &diff.present {
+        let stored = stored_keywords
+            .get(local_id)
+            .map(String::as_str)
+            .unwrap_or("[]");
+        match refresh_flags(&tx, *local_id, entry, stored, ctx.include_deleted) {
+            Ok(PresentOutcome::Unchanged) => counts.skipped += 1,
+            Ok(PresentOutcome::KeywordsUpdated) => counts.fetched += 1,
+            Ok(PresentOutcome::DeleteRequested) => {
+                match delete_vanished(&tx, ctx.source_id, ctx.folder, &entry.unique_id, *local_id) {
+                    Ok(()) => counts.deleted += 1,
+                    Err(e) => {
+                        logger.warn(&format!(
+                            "maildir {folder:?}/{name}: T-flag delete failed: {e}",
+                            folder = ctx.folder,
+                            name = entry.filename
+                        ));
+                        counts.failed += 1;
+                    }
+                }
+            }
+            Err(e) => {
+                logger.warn(&format!(
+                    "maildir {folder:?}/{name}: flag update failed: {e}",
+                    folder = ctx.folder,
+                    name = entry.filename
+                ));
+                counts.failed += 1;
+            }
+        }
+    }
+    for (unique_id, local_id) in &diff.vanished {
+        if let Err(e) = delete_vanished(&tx, ctx.source_id, ctx.folder, unique_id, *local_id) {
+            logger.warn(&format!(
+                "maildir {folder:?}/{unique_id}: delete failed: {e}",
+                folder = ctx.folder
+            ));
+            counts.failed += 1;
+        } else {
+            counts.deleted += 1;
+        }
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+fn keywords_json(translation: &Translation) -> String {
+    Value::Array(
+        translation
+            .keywords
+            .iter()
+            .map(|k| Value::String(k.clone()))
+            .collect(),
+    )
+    .to_string()
+}
+
+fn format_unix_rfc3339(secs: u64) -> String {
+    let when = UNIX_EPOCH + Duration::from_secs(secs);
+    OffsetDateTime::from(when)
+        .format(&Rfc3339)
+        .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned())
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum InsertError {
+    #[error("io: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("db: {0}")]
+    Db(#[from] rusqlite::Error),
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::db::init;
+    use crate::db::sources::{SourceKey, upsert_source};
+    use std::fs;
+
+    fn fresh_archive() -> (Connection, i64) {
+        let c = Connection::open_in_memory().unwrap();
+        init::apply_schema(&c).unwrap();
+        let sid = upsert_source(
+            &c,
+            &SourceKey {
+                kind: "maildir".to_owned(),
+                session_url: "file:///tmp/Maildir".to_owned(),
+                account_id: "/tmp/Maildir".to_owned(),
+            },
+            Some("Maildir"),
+            "",
+        )
+        .unwrap();
+        (c, sid)
+    }
+
+    fn write_maildir_message(
+        folder_path: &Path,
+        sub: &str,
+        filename: &str,
+        body: &[u8],
+    ) -> PathBuf {
+        let dir = folder_path.join(sub);
+        fs::create_dir_all(&dir).unwrap();
+        let p = dir.join(filename);
+        fs::write(&p, body).unwrap();
+        p
+    }
+
+    fn ensure_folder_skel(folder_path: &Path) {
+        for sub in ["cur", "new", "tmp"] {
+            fs::create_dir_all(folder_path.join(sub)).unwrap();
+        }
+    }
+
+    #[test]
+    fn list_folder_returns_cur_and_new_skipping_tmp() {
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a");
+        write_maildir_message(td.path(), "new", "2.M0.host", b"b");
+        write_maildir_message(td.path(), "tmp", "3.M0.host", b"c");
+        let listing = list_folder(td.path()).unwrap();
+        assert_eq!(listing.entries.len(), 2);
+        assert!(listing.entries.iter().any(|e| e.unique_id == "1.M0.host"));
+        assert!(listing.entries.iter().any(|e| e.unique_id == "2.M0.host"));
+    }
+
+    #[test]
+    fn list_folder_skips_dotfiles() {
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", ".uidvalidity", b"meta");
+        write_maildir_message(td.path(), "cur", ".dovecot-uidlist", b"meta");
+        write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a");
+        let listing = list_folder(td.path()).unwrap();
+        assert_eq!(listing.entries.len(), 1);
+        assert_eq!(listing.entries[0].unique_id, "1.M0.host");
+    }
+
+    #[test]
+    fn list_folder_parses_dovecot_extension_filename() {
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(
+            td.path(),
+            "cur",
+            "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44:2,RS",
+            b"a",
+        );
+        let listing = list_folder(td.path()).unwrap();
+        assert_eq!(listing.entries.len(), 1);
+        let e = &listing.entries[0];
+        assert_eq!(
+            e.unique_id,
+            "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44"
+        );
+        assert!(e.flags.contains(&Flag::Replied));
+        assert!(e.flags.contains(&Flag::Seen));
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn list_folder_renders_non_utf8_filename_lossy() {
+        use std::ffi::OsStr;
+        use std::os::unix::ffi::OsStrExt;
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        let mut bytes = b"1.M0.host".to_vec();
+        bytes.extend_from_slice(&[0xff, 0xfe]);
+        bytes.extend_from_slice(b":2,S");
+        let bad_name = OsStr::from_bytes(&bytes);
+        let p = td.path().join("cur").join(bad_name);
+        fs::write(&p, b"body").unwrap();
+        let listing = list_folder(td.path()).unwrap();
+        assert_eq!(listing.entries.len(), 1);
+        let e = &listing.entries[0];
+        assert!(
+            e.unique_id.contains('\u{FFFD}'),
+            "lossy substitution: {:?}",
+            e.unique_id
+        );
+        assert!(e.unique_id.starts_with("1.M0.host"));
+        assert!(e.flags.contains(&Flag::Seen));
+    }
+
+    #[test]
+    fn diff_partitions_new_present_vanished() {
+        let entries = vec![
+            DiskEntry {
+                unique_id: "a".into(),
+                filename: "a".into(),
+                path: PathBuf::new(),
+                flags: vec![],
+                mtime_unix: 0,
+            },
+            DiskEntry {
+                unique_id: "b".into(),
+                filename: "b".into(),
+                path: PathBuf::new(),
+                flags: vec![],
+                mtime_unix: 0,
+            },
+        ];
+        let mut local = HashMap::new();
+        local.insert("b".to_owned(), 10);
+        local.insert("c".to_owned(), 11);
+        let d = diff(entries, &local);
+        assert_eq!(d.new.len(), 1);
+        assert_eq!(d.new[0].unique_id, "a");
+        assert_eq!(d.present.len(), 1);
+        assert_eq!(d.present[0].0.unique_id, "b");
+        assert_eq!(d.present[0].1, 10);
+        assert_eq!(d.vanished, vec![("c".to_owned(), 11)]);
+    }
+
+    #[test]
+    fn insert_new_writes_blob_email_and_sync_row() {
+        let (mut c, sid) = fresh_archive();
+        c.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+             VALUES ('INBOX', NULL, 'inbox', 0, 1)",
+            [],
+        )
+        .unwrap();
+        let mailbox_local: i64 = c.last_insert_rowid();
+        db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap();
+
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        let body = b"From: a@x\r\nSubject: hi\r\nMessage-ID: <m1@h>\r\n\r\nhi";
+        write_maildir_message(td.path(), "cur", "uid1.M0.host:2,S", body);
+        let listing = list_folder(td.path()).unwrap();
+        let entry = listing.entries.into_iter().next().unwrap();
+
+        let tx = c.transaction().unwrap();
+        let inserted = insert_new(
+            &tx,
+            InsertContext {
+                source_id: sid,
+                folder: "INBOX",
+                mailbox_local,
+                include_deleted: false,
+            },
+            &entry,
+        )
+        .unwrap();
+        tx.commit().unwrap();
+        assert!(inserted.is_some());
+
+        let email_count: i64 = c
+            .query_row("SELECT count(*) FROM emails", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(email_count, 1);
+        let local = db::maildir_ids::local_for_email(&c, sid, "INBOX", "uid1.M0.host")
+            .unwrap()
+            .unwrap();
+        assert_eq!(local, inserted.unwrap());
+        let kws: String = c
+            .query_row(
+                "SELECT keywords FROM emails WHERE id = ?1",
+                params![local],
+                |r| r.get(0),
+            )
+            .unwrap();
+        assert!(kws.contains("$seen"));
+    }
+
+    #[test]
+    fn insert_new_skips_trashed_without_include_deleted() {
+        let (mut c, sid) = fresh_archive();
+        c.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+             VALUES ('INBOX', NULL, 'inbox', 0, 1)",
+            [],
+        )
+        .unwrap();
+        let mailbox_local: i64 = c.last_insert_rowid();
+        db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap();
+
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", "uid.host:2,T", b"x");
+        let listing = list_folder(td.path()).unwrap();
+        let entry = listing.entries.into_iter().next().unwrap();
+        let tx = c.transaction().unwrap();
+        let inserted = insert_new(
+            &tx,
+            InsertContext {
+                source_id: sid,
+                folder: "INBOX",
+                mailbox_local,
+                include_deleted: false,
+            },
+            &entry,
+        )
+        .unwrap();
+        tx.commit().unwrap();
+        assert!(inserted.is_none());
+        let email_count: i64 = c
+            .query_row("SELECT count(*) FROM emails", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(email_count, 0);
+    }
+
+    #[test]
+    fn refresh_flags_updates_only_when_changed() {
+        let (mut c, sid) = fresh_archive();
+        c.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+             VALUES ('INBOX', NULL, 'inbox', 0, 1)",
+            [],
+        )
+        .unwrap();
+        let mailbox_local = c.last_insert_rowid();
+        db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap();
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", "u.host:2,", b"hi");
+        let entry = list_folder(td.path()).unwrap().entries.remove(0);
+        let tx = c.transaction().unwrap();
+        let local = insert_new(
+            &tx,
+            InsertContext {
+                source_id: sid,
+                folder: "INBOX",
+                mailbox_local,
+                include_deleted: false,
+            },
+            &entry,
+        )
+        .unwrap()
+        .unwrap();
+        tx.commit().unwrap();
+
+        let stored = load_present_keywords(&c, sid, "INBOX").unwrap();
+        let stored_for_local = stored
+            .get(&local)
+            .cloned()
+            .unwrap_or_else(|| "[]".to_owned());
+
+        let tx = c.transaction().unwrap();
+        assert_eq!(
+            refresh_flags(&tx, local, &entry, &stored_for_local, false).unwrap(),
+            PresentOutcome::Unchanged
+        );
+        tx.commit().unwrap();
+
+        fs::rename(&entry.path, entry.path.with_file_name("u.host:2,S")).unwrap();
+        let entry2 = list_folder(td.path()).unwrap().entries.remove(0);
+        let tx = c.transaction().unwrap();
+        assert_eq!(
+            refresh_flags(&tx, local, &entry2, &stored_for_local, false).unwrap(),
+            PresentOutcome::KeywordsUpdated
+        );
+        tx.commit().unwrap();
+        let kws: String = c
+            .query_row(
+                "SELECT keywords FROM emails WHERE id = ?1",
+                params![local],
+                |r| r.get(0),
+            )
+            .unwrap();
+        assert!(kws.contains("$seen"));
+    }
+
+    #[test]
+    fn refresh_flags_returns_delete_when_trashed_added_and_include_deleted_off() {
+        let (mut c, sid) = fresh_archive();
+        c.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+             VALUES ('INBOX', NULL, 'inbox', 0, 1)",
+            [],
+        )
+        .unwrap();
+        let mailbox_local = c.last_insert_rowid();
+        db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap();
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", "u.host:2,", b"hi");
+        let entry = list_folder(td.path()).unwrap().entries.remove(0);
+        let tx = c.transaction().unwrap();
+        let local = insert_new(
+            &tx,
+            InsertContext {
+                source_id: sid,
+                folder: "INBOX",
+                mailbox_local,
+                include_deleted: false,
+            },
+            &entry,
+        )
+        .unwrap()
+        .unwrap();
+        tx.commit().unwrap();
+
+        fs::rename(&entry.path, entry.path.with_file_name("u.host:2,T")).unwrap();
+        let entry2 = list_folder(td.path()).unwrap().entries.remove(0);
+        let tx = c.transaction().unwrap();
+        assert_eq!(
+            refresh_flags(&tx, local, &entry2, "[]", false).unwrap(),
+            PresentOutcome::DeleteRequested
+        );
+        tx.commit().unwrap();
+    }
+
+    #[test]
+    fn refresh_flags_keeps_trashed_with_include_deleted_on() {
+        let (mut c, sid) = fresh_archive();
+        c.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+             VALUES ('INBOX', NULL, 'inbox', 0, 1)",
+            [],
+        )
+        .unwrap();
+        let mailbox_local = c.last_insert_rowid();
+        db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap();
+        let td = tempfile::tempdir().unwrap();
+        ensure_folder_skel(td.path());
+        write_maildir_message(td.path(), "cur", "u.host:2,T", b"hi");
+        let entry = list_folder(td.path()).unwrap().entries.remove(0);
+        let tx = c.transaction().unwrap();
+        let local = insert_new(
+            &tx,
+            InsertContext {
+                source_id: sid,
+                folder: "INBOX",
+                mailbox_local,
+                include_deleted: true,
+            },
+            &entry,
+        )
+        .unwrap()
+        .unwrap();
+        tx.commit().unwrap();
+
+        let stored = load_present_keywords(&c, sid, "INBOX").unwrap();
+        let stored_for_local = stored.get(&local).cloned().unwrap();
+        let tx = c.transaction().unwrap();
+        assert_eq!(
+            refresh_flags(&tx, local, &entry, &stored_for_local, true).unwrap(),
+            PresentOutcome::Unchanged
+        );
+        tx.commit().unwrap();
+    }
+}

src/sync/import_maildir/tree.rs

@@ -0,0 +1,495 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::path::{Path, PathBuf};
+
+use regex::Regex;
+
+use crate::imap::automap;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ResolvedFolder {
+    pub name: String,
+    pub leaf: String,
+    pub parent_path: Option<String>,
+    pub path: PathBuf,
+    pub role: Option<&'static str>,
+    pub ephemeral: bool,
+}
+
+pub struct FolderFilters {
+    pub include: Vec<Regex>,
+    pub exclude: Vec<Regex>,
+    pub explicit: Vec<String>,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum DiscoverError {
+    #[error("path does not exist: {0}")]
+    NotFound(PathBuf),
+    #[error("path is not a directory: {0}")]
+    NotADirectory(PathBuf),
+    #[error("not a Maildir: {0} (missing cur/ subdirectory)")]
+    NotAMaildir(PathBuf),
+    #[error(
+        "only Maildir++ layout is supported: found directory {0:?} that looks like a non-prefixed \
+         subfolder (subfolders must be named with a leading '.', e.g. '.Sent')"
+    )]
+    NotMaildirPlus(String),
+    #[error("io error walking {0}: {1}")]
+    Io(PathBuf, std::io::Error),
+}
+
+pub fn discover(root: &Path, automap_enabled: bool) -> Result<Vec<ResolvedFolder>, DiscoverError> {
+    if !root.exists() {
+        return Err(DiscoverError::NotFound(root.to_path_buf()));
+    }
+    let meta = std::fs::metadata(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?;
+    if !meta.is_dir() {
+        return Err(DiscoverError::NotADirectory(root.to_path_buf()));
+    }
+    if !root.join("cur").is_dir() {
+        return Err(DiscoverError::NotAMaildir(root.to_path_buf()));
+    }
+    let mut folders = vec![ResolvedFolder {
+        name: "INBOX".to_owned(),
+        leaf: "INBOX".to_owned(),
+        parent_path: None,
+        path: root.to_path_buf(),
+        role: Some("inbox"),
+        ephemeral: false,
+    }];
+
+    let mut subfolders: Vec<(String, PathBuf)> = Vec::new();
+    let entries = std::fs::read_dir(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?;
+    for entry in entries {
+        let entry = entry.map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?;
+        let path = entry.path();
+        if !path.is_dir() {
+            continue;
+        }
+        let Some(raw_name) = path.file_name() else {
+            continue;
+        };
+        let name: String = raw_name.to_string_lossy().into_owned();
+        if matches!(name.as_str(), "cur" | "new" | "tmp") {
+            continue;
+        }
+        let has_cur = path.join("cur").is_dir();
+        if let Some(stripped) = name.strip_prefix('.') {
+            if has_cur && !stripped.is_empty() {
+                subfolders.push((stripped.to_owned(), path));
+            }
+        } else if has_cur {
+            return Err(DiscoverError::NotMaildirPlus(name));
+        }
+    }
+
+    subfolders.sort_by(|a, b| a.0.cmp(&b.0));
+
+    let mut known: Vec<String> = vec!["INBOX".to_owned()];
+    for (canonical, path) in subfolders {
+        let parent_path = split_parent(&canonical);
+        if let Some(p) = parent_path.as_deref() {
+            ensure_ephemeral_parents(p, &mut known, &mut folders, root);
+        }
+        let leaf = leaf_of(&canonical);
+        let role = if automap_enabled {
+            automap::role_for_folder(&canonical, &[], "", true)
+        } else {
+            None
+        };
+        folders.push(ResolvedFolder {
+            name: canonical.clone(),
+            leaf: leaf.to_owned(),
+            parent_path,
+            path,
+            role,
+            ephemeral: false,
+        });
+        known.push(canonical);
+    }
+    Ok(folders)
+}
+
+fn ensure_ephemeral_parents(
+    parent_path: &str,
+    known: &mut Vec<String>,
+    folders: &mut Vec<ResolvedFolder>,
+    root: &Path,
+) {
+    for ancestor in ancestor_chain(parent_path) {
+        if known.iter().any(|n| n == &ancestor) {
+            continue;
+        }
+        let leaf = leaf_of(&ancestor).to_owned();
+        let parent_path = split_parent(&ancestor);
+        folders.push(ResolvedFolder {
+            name: ancestor.clone(),
+            leaf,
+            parent_path,
+            path: root.to_path_buf(),
+            role: None,
+            ephemeral: true,
+        });
+        known.push(ancestor);
+    }
+}
+
+fn ancestor_chain(name: &str) -> Vec<String> {
+    let mut out = Vec::new();
+    let mut acc = String::new();
+    for part in name.split('.') {
+        if !acc.is_empty() {
+            acc.push('.');
+        }
+        acc.push_str(part);
+        out.push(acc.clone());
+    }
+    out
+}
+
+fn split_parent(name: &str) -> Option<String> {
+    name.rfind('.').map(|i| name[..i].to_owned())
+}
+
+fn leaf_of(name: &str) -> &str {
+    match name.rfind('.') {
+        Some(i) => &name[i + 1..],
+        None => name,
+    }
+}
+
+pub fn apply_filters(folders: Vec<ResolvedFolder>, filters: &FolderFilters) -> Vec<ResolvedFolder> {
+    folders
+        .into_iter()
+        .filter(|f| match_filters(&f.name, filters))
+        .collect()
+}
+
+pub fn restore_ephemeral_parents(folders: &mut Vec<ResolvedFolder>, root: &Path) {
+    let known: std::collections::HashSet<String> = folders.iter().map(|f| f.name.clone()).collect();
+    let mut to_add: Vec<ResolvedFolder> = Vec::new();
+    for f in folders.iter() {
+        if let Some(parent) = f.parent_path.as_deref() {
+            for ancestor in ancestor_chain(parent) {
+                if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) {
+                    continue;
+                }
+                let leaf = leaf_of(&ancestor).to_owned();
+                let parent_path = split_parent(&ancestor);
+                to_add.push(ResolvedFolder {
+                    name: ancestor,
+                    leaf,
+                    parent_path,
+                    path: root.to_path_buf(),
+                    role: None,
+                    ephemeral: true,
+                });
+            }
+        }
+    }
+    if to_add.is_empty() {
+        return;
+    }
+    folders.extend(to_add);
+    folders.sort_by_key(|f| f.name.matches('.').count());
+}
+
+fn match_filters(name: &str, filters: &FolderFilters) -> bool {
+    if !filters.explicit.is_empty() {
+        return filters.explicit.iter().any(|e| e == name);
+    }
+    if !filters.include.is_empty() && !filters.include.iter().any(|r| r.is_match(name)) {
+        return false;
+    }
+    if filters.exclude.iter().any(|r| r.is_match(name)) {
+        return false;
+    }
+    true
+}
+
+pub fn vanished_folders<'a, I>(local: I, server: &[ResolvedFolder]) -> Vec<String>
+where
+    I: IntoIterator<Item = &'a str>,
+{
+    let server_set: std::collections::HashSet<&str> =
+        server.iter().map(|f| f.name.as_str()).collect();
+    local
+        .into_iter()
+        .filter(|n| !server_set.contains(*n))
+        .map(str::to_owned)
+        .collect()
+}
+
+pub fn vanished_depth_sort(names: &mut [String]) {
+    names.sort_by(|a, b| {
+        let da = a.matches('.').count();
+        let db = b.matches('.').count();
+        db.cmp(&da).then_with(|| a.cmp(b))
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+
+    fn make_maildir(root: &Path, folders: &[&str]) {
+        for sub in ["cur", "new", "tmp"] {
+            fs::create_dir_all(root.join(sub)).unwrap();
+        }
+        for name in folders {
+            let p = root.join(name);
+            for sub in ["cur", "new", "tmp"] {
+                fs::create_dir_all(p.join(sub)).unwrap();
+            }
+        }
+    }
+
+    #[test]
+    fn discover_just_inbox_when_root_alone() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[]);
+        let folders = discover(td.path(), true).unwrap();
+        assert_eq!(folders.len(), 1);
+        assert_eq!(folders[0].name, "INBOX");
+        assert_eq!(folders[0].role, Some("inbox"));
+        assert!(folders[0].parent_path.is_none());
+    }
+
+    #[test]
+    fn discover_dovecot_style_subfolders() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[".Sent", ".Drafts", ".Trash", ".Junk"]);
+        let folders = discover(td.path(), true).unwrap();
+        let names: Vec<&str> = folders.iter().map(|f| f.name.as_str()).collect();
+        assert!(names.contains(&"INBOX"));
+        assert!(names.contains(&"Sent"));
+        assert!(names.contains(&"Drafts"));
+        let sent = folders.iter().find(|f| f.name == "Sent").unwrap();
+        assert_eq!(sent.role, Some("sent"));
+        let drafts = folders.iter().find(|f| f.name == "Drafts").unwrap();
+        assert_eq!(drafts.role, Some("drafts"));
+        let trash = folders.iter().find(|f| f.name == "Trash").unwrap();
+        assert_eq!(trash.role, Some("trash"));
+        let junk = folders.iter().find(|f| f.name == "Junk").unwrap();
+        assert_eq!(junk.role, Some("junk"));
+    }
+
+    #[test]
+    fn discover_courier_maildir_plus_deep_hierarchy() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[".Archive", ".Archive.2024", ".Archive.2025"]);
+        let folders = discover(td.path(), true).unwrap();
+        let names: Vec<&str> = folders.iter().map(|f| f.name.as_str()).collect();
+        assert!(names.contains(&"Archive"));
+        assert!(names.contains(&"Archive.2024"));
+        assert!(names.contains(&"Archive.2025"));
+        let a2025 = folders.iter().find(|f| f.name == "Archive.2025").unwrap();
+        assert_eq!(a2025.leaf, "2025");
+        assert_eq!(a2025.parent_path.as_deref(), Some("Archive"));
+        let archive = folders.iter().find(|f| f.name == "Archive").unwrap();
+        assert_eq!(archive.role, Some("archive"));
+
+        assert!(a2025.role.is_none());
+    }
+
+    #[test]
+    fn discover_inserts_ephemeral_parent_for_orphan_branch() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[".Lists.maildir-dev"]);
+        let folders = discover(td.path(), true).unwrap();
+        let lists = folders.iter().find(|f| f.name == "Lists").unwrap();
+        assert!(lists.ephemeral);
+        assert!(lists.role.is_none());
+        assert!(lists.parent_path.is_none());
+        let leaf = folders
+            .iter()
+            .find(|f| f.name == "Lists.maildir-dev")
+            .unwrap();
+        assert!(!leaf.ephemeral);
+        assert_eq!(leaf.parent_path.as_deref(), Some("Lists"));
+        assert_eq!(leaf.leaf, "maildir-dev");
+    }
+
+    #[test]
+    fn discover_skips_non_dot_subfolders_silently_only_if_no_cur() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[".Sent"]);
+
+        fs::create_dir(td.path().join(".dovecot.imap")).unwrap();
+        fs::write(td.path().join("dovecot.index.log"), "").unwrap();
+        let folders = discover(td.path(), true).unwrap();
+        assert_eq!(folders.len(), 2);
+        assert!(folders.iter().any(|f| f.name == "INBOX"));
+        assert!(folders.iter().any(|f| f.name == "Sent"));
+    }
+
+    #[test]
+    fn discover_rejects_dovecot_layout_fs_tree() {
+
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &["Sent"]);
+        let err = discover(td.path(), true).unwrap_err();
+        assert!(matches!(err, DiscoverError::NotMaildirPlus(_)));
+    }
+
+    #[test]
+    fn discover_rejects_path_without_cur_subdir() {
+        let td = tempfile::tempdir().unwrap();
+        fs::create_dir_all(td.path().join("new")).unwrap();
+        let err = discover(td.path(), true).unwrap_err();
+        assert!(matches!(err, DiscoverError::NotAMaildir(_)));
+    }
+
+    #[test]
+    fn discover_rejects_nonexistent_path() {
+        let err = discover(Path::new("/definitely/not/a/maildir/at/all"), true).unwrap_err();
+        assert!(matches!(err, DiscoverError::NotFound(_)));
+    }
+
+    #[test]
+    fn discover_returns_parents_before_children() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(
+            td.path(),
+            &[".Archive.2025", ".Archive", ".Lists.maildir-dev"],
+        );
+        let folders = discover(td.path(), true).unwrap();
+        let index = |name: &str| folders.iter().position(|f| f.name == name).unwrap();
+        assert!(index("Archive") < index("Archive.2025"));
+        assert!(index("Lists") < index("Lists.maildir-dev"));
+        assert_eq!(index("INBOX"), 0);
+    }
+
+    #[test]
+    fn ancestor_chain_walks_dot_segments() {
+        assert_eq!(ancestor_chain("A"), vec!["A".to_owned()]);
+        assert_eq!(
+            ancestor_chain("A.B.C"),
+            vec!["A".to_owned(), "A.B".to_owned(), "A.B.C".to_owned()]
+        );
+    }
+
+    #[test]
+    fn leaf_and_split_handle_root_and_nested() {
+        assert_eq!(leaf_of("INBOX"), "INBOX");
+        assert_eq!(leaf_of("Archive.2025"), "2025");
+        assert_eq!(split_parent("INBOX"), None);
+        assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned()));
+    }
+
+    fn folders_for(names: &[&str]) -> Vec<ResolvedFolder> {
+        names
+            .iter()
+            .map(|n| ResolvedFolder {
+                name: (*n).to_owned(),
+                leaf: (*n).to_owned(),
+                parent_path: None,
+                path: PathBuf::new(),
+                role: None,
+                ephemeral: false,
+            })
+            .collect()
+    }
+
+    #[test]
+    fn include_filter_keeps_only_matches() {
+        let folders = folders_for(&["INBOX", "Sent", "Trash"]);
+        let filters = FolderFilters {
+            include: vec![Regex::new("^(INBOX|Sent)$").unwrap()],
+            exclude: Vec::new(),
+            explicit: Vec::new(),
+        };
+        let kept = apply_filters(folders, &filters);
+        assert_eq!(kept.len(), 2);
+        assert!(kept.iter().any(|f| f.name == "INBOX"));
+        assert!(kept.iter().any(|f| f.name == "Sent"));
+    }
+
+    #[test]
+    fn exclude_filter_drops_matches() {
+        let folders = folders_for(&["INBOX", "Trash"]);
+        let filters = FolderFilters {
+            include: Vec::new(),
+            exclude: vec![Regex::new("^Trash$").unwrap()],
+            explicit: Vec::new(),
+        };
+        let kept = apply_filters(folders, &filters);
+        assert_eq!(kept.len(), 1);
+        assert_eq!(kept[0].name, "INBOX");
+    }
+
+    #[test]
+    fn explicit_folder_overrides_include_exclude() {
+        let folders = folders_for(&["INBOX", "Sent", "Trash"]);
+        let filters = FolderFilters {
+            include: Vec::new(),
+            exclude: Vec::new(),
+            explicit: vec!["Sent".to_owned()],
+        };
+        let kept = apply_filters(folders, &filters);
+        assert_eq!(kept.len(), 1);
+        assert_eq!(kept[0].name, "Sent");
+    }
+
+    #[test]
+    fn discover_with_automap_off_clears_roles_except_inbox() {
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[".Sent", ".Drafts"]);
+        let folders = discover(td.path(), false).unwrap();
+        let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap();
+        assert_eq!(inbox.role, Some("inbox"));
+        let sent = folders.iter().find(|f| f.name == "Sent").unwrap();
+        assert_eq!(sent.role, None);
+        let drafts = folders.iter().find(|f| f.name == "Drafts").unwrap();
+        assert_eq!(drafts.role, None);
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn discover_renders_non_utf8_folder_name_lossy() {
+        use std::ffi::OsStr;
+        use std::os::unix::ffi::OsStrExt;
+        let td = tempfile::tempdir().unwrap();
+        make_maildir(td.path(), &[]);
+        let mut bytes = b".Foo".to_vec();
+        bytes.extend_from_slice(&[0xff, 0xfe]);
+        let dotted = OsStr::from_bytes(&bytes);
+        let p = td.path().join(dotted);
+        for sub in ["cur", "new", "tmp"] {
+            fs::create_dir_all(p.join(sub)).unwrap();
+        }
+        let folders = discover(td.path(), true).unwrap();
+        assert_eq!(folders.len(), 2);
+        let sub = folders.iter().find(|f| f.name != "INBOX").unwrap();
+        assert!(
+            sub.name.contains('\u{FFFD}'),
+            "lossy substitution: {:?}",
+            sub.name
+        );
+        assert!(sub.name.starts_with("Foo"));
+    }
+
+    #[test]
+    fn vanished_depth_sort_deepest_first() {
+        let mut names = vec![
+            "Archive".to_owned(),
+            "Archive.2025.January".to_owned(),
+            "Archive.2025".to_owned(),
+        ];
+        vanished_depth_sort(&mut names);
+        assert_eq!(
+            names,
+            vec![
+                "Archive.2025.January".to_owned(),
+                "Archive.2025".to_owned(),
+                "Archive".to_owned(),
+            ]
+        );
+    }
+}

src/sync/import_managesieve.rs

@@ -0,0 +1,10 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod coordinator;
+pub mod reconcile;
+
+pub use coordinator::{ManageSieveAuth, ManageSieveImportConfig, run};

src/sync/import_managesieve/coordinator.rs

@@ -0,0 +1,638 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::sync::Arc;
+
+use url::Url;
+
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::error::Error;
+use crate::imap::retry::BackoffState;
+use crate::imap::transport::Connector;
+use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
+use crate::managesieve::error::SieveError;
+use crate::managesieve::response::parse_listscripts;
+use crate::managesieve::retry::{Disposition, classify, is_negotiation_failure};
+use crate::managesieve::{ConnectMode, SieveClient, parse_getscript};
+use crate::sync::keys::blake3_bytes;
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+
+use super::reconcile::{
+    Action, PresentOutcome, apply_active_assignment, apply_content_update, apply_delete, apply_new,
+    load_local_state, plan,
+};
+
+pub struct ManageSieveImportConfig {
+    pub url: String,
+    pub auth: ManageSieveAuth,
+    pub allow_cleartext: bool,
+    pub allow_source_change: bool,
+}
+
+#[derive(Debug, Clone)]
+pub enum ManageSieveAuth {
+    Basic { user: String, password: String },
+    Bearer { user: String, token: String },
+}
+
+#[derive(Debug, Clone)]
+struct Endpoint {
+    host: String,
+    port: u16,
+    implicit_tls: bool,
+}
+
+struct ControlCtx {
+    connector: Arc<Connector>,
+    host: String,
+    port: u16,
+    mode: ConnectMode,
+    allow_cleartext: bool,
+    auth: ManageSieveAuth,
+    backoff: BackoffState,
+    max_retries: u32,
+    logger: Logger,
+}
+
+fn reconnect(client: &mut SieveClient, ctx: &ControlCtx) -> Result<(), SieveError> {
+    let new = SieveClient::connect(
+        &ctx.connector,
+        &ctx.host,
+        ctx.port,
+        ctx.mode,
+        ctx.allow_cleartext,
+    )?;
+    *client = new;
+    match do_authenticate(client, &ctx.auth) {
+        Ok(()) => {}
+        Err(SieveAuthError::Wire(e)) => return Err(e),
+        Err(SieveAuthError::TerminallyRefused(text)) => {
+            return Err(SieveError::Protocol(format!(
+                "auth refused on reconnect: {text}"
+            )));
+        }
+        Err(SieveAuthError::NoUsableMechanism(text)) => {
+            return Err(SieveError::Unsupported(text));
+        }
+    }
+    if !client.had_fresh_post_auth_caps()
+        && let Err(e) = client.refresh_capabilities()
+    {
+        ctx.logger.warn(&format!(
+            "post-reconnect CAPABILITY refresh failed: {e}; continuing"
+        ));
+    }
+    Ok(())
+}
+
+pub fn run(common: CommonConfig, config: ManageSieveImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    if common.threads > 1 {
+        log_at(
+            logger,
+            LEVEL_PROGRESS,
+            "managesieve importer is single-threaded; --threads value will be ignored",
+        );
+    }
+
+    let mut conn = db::init::open(&common.archive)?;
+
+    let endpoint = parse_endpoint(&config.url)?;
+    let session_url = format!(
+        "{}://{}:{}",
+        if endpoint.implicit_tls {
+            "sieves"
+        } else {
+            "sieve"
+        },
+        endpoint.host,
+        endpoint.port
+    );
+
+    let connector = Arc::new(
+        Connector::new(common.allow_invalid_certs).map_err(|e| Error::Connection(e.to_string()))?,
+    );
+    let mode = if endpoint.implicit_tls {
+        ConnectMode::ImplicitTls
+    } else {
+        ConnectMode::StartTls
+    };
+
+    let mut client = SieveClient::connect(
+        &connector,
+        &endpoint.host,
+        endpoint.port,
+        mode,
+        config.allow_cleartext,
+    )
+    .map_err(|e| Error::Connection(e.to_string()))?;
+
+    let account_id = authenticate(&mut client, &config.auth)?;
+    log_at(
+        logger,
+        LEVEL_PROGRESS,
+        &format!("authenticated as {account_id} on {session_url}"),
+    );
+    if !client.had_fresh_post_auth_caps()
+        && let Err(e) = client.refresh_capabilities()
+    {
+        logger.warn(&format!(
+            "post-auth CAPABILITY refresh failed: {e}; continuing"
+        ));
+    }
+
+    let source_key = SourceKey {
+        kind: "managesieve".to_owned(),
+        session_url: session_url.clone(),
+        account_id: account_id.clone(),
+    };
+    if let Some((existing_url, existing_account)) =
+        db::sources::conflicting_source(&conn, "managesieve", &session_url, &account_id)?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records managesieve source {existing_url} / {existing_account}; \
+             re-run with --allow-source-change or use a fresh archive"
+        )));
+    }
+    let source_id = if common.dry_run {
+        db::sources::find_source(&conn, &source_key)?.unwrap_or(-1)
+    } else {
+        db::sources::upsert_source(&conn, &source_key, Some(&account_id), &account_id)?
+    };
+
+    let backoff = BackoffState::new();
+    let control_ctx = ControlCtx {
+        connector: connector.clone(),
+        host: endpoint.host.clone(),
+        port: endpoint.port,
+        mode,
+        allow_cleartext: config.allow_cleartext,
+        auth: config.auth.clone(),
+        backoff: backoff.clone(),
+        max_retries: common.max_retries,
+        logger,
+    };
+    let server_listing = listscripts_with_retry(&mut client, &control_ctx)?;
+
+    let mut server_with_bytes: Vec<(String, bool, [u8; 32], Vec<u8>)> =
+        Vec::with_capacity(server_listing.len());
+    let mut counts = TypeCounts::default();
+    for (name, active) in &server_listing {
+        match getscript_with_retry(&mut client, &control_ctx, name) {
+            Ok(bytes) => {
+                let hash = blake3_bytes(&bytes);
+                server_with_bytes.push((name.clone(), *active, hash, bytes));
+            }
+            Err(e) => match classify(&e) {
+                Disposition::Permanent
+                | Disposition::PerScriptRecoverable
+                | Disposition::Transient => {
+                    logger.warn(&format!("GETSCRIPT {name:?} failed, skipping: {e}"));
+                    counts.failed += 1;
+                }
+                Disposition::Referral => {
+                    return Err(Error::Connection(format!(
+                        "ManageSieve referral on GETSCRIPT {name:?}: {e}; vandelay does not follow referrals"
+                    )));
+                }
+                Disposition::TransportDrop => {
+                    return Err(Error::Connection(format!(
+                        "GETSCRIPT {name:?} transport drop exhausted retries: {e}"
+                    )));
+                }
+                Disposition::FeatureNegotiation => {
+                    return Err(Error::Connection(format!("GETSCRIPT {name:?}: {e}")));
+                }
+            },
+        }
+    }
+
+    if common.dry_run {
+        let summary =
+            build_dry_run_summary(&conn, source_id, &server_with_bytes, counts, &backoff)?;
+        let _ = client.logout();
+        return Ok(summary);
+    }
+
+    let local_map = db::managesieve_ids::all_names(&conn, source_id)?;
+    let (local_hashes, local_active) = load_local_state(&conn, source_id)?;
+    let server_keyed: Vec<(String, bool, [u8; 32])> = server_with_bytes
+        .iter()
+        .map(|(n, a, h, _)| (n.clone(), *a, *h))
+        .collect();
+    let plan = plan(&server_keyed, &local_map, &local_hashes, &local_active);
+
+    let bytes_by_name: std::collections::HashMap<String, &Vec<u8>> = server_with_bytes
+        .iter()
+        .map(|(n, _, _, b)| (n.clone(), b))
+        .collect();
+
+    let mut active_local: Option<i64> = None;
+
+    for action in &plan.actions {
+        match action {
+            Action::New { name, active } => {
+                let bytes = bytes_by_name.get(name).cloned().unwrap();
+                let tx = conn.transaction()?;
+                match apply_new(&tx, source_id, name, bytes) {
+                    Ok(local_id) => {
+                        tx.commit()?;
+                        if *active {
+                            active_local = Some(local_id);
+                        }
+                        counts.created += 1;
+                        counts.fetched += 1;
+                    }
+                    Err(e) => {
+                        let _ = tx.rollback();
+                        logger.warn(&format!("INSERT sieve_scripts {name:?} failed: {e}"));
+                        counts.failed += 1;
+                    }
+                }
+            }
+            Action::Present {
+                name,
+                local_id,
+                outcome,
+                active,
+            } => {
+                match outcome {
+                    PresentOutcome::Unchanged => {
+                        counts.skipped += 1;
+                    }
+                    PresentOutcome::ActiveOnly => {
+                        counts.skipped += 1;
+                    }
+                    PresentOutcome::ContentUpdated => {
+                        let bytes = bytes_by_name.get(name).cloned().unwrap();
+                        let tx = conn.transaction()?;
+                        match apply_content_update(&tx, *local_id, bytes) {
+                            Ok(()) => {
+                                tx.commit()?;
+                                counts.fetched += 1;
+                            }
+                            Err(e) => {
+                                let _ = tx.rollback();
+                                logger.warn(&format!("UPDATE sieve_scripts {name:?} failed: {e}"));
+                                counts.failed += 1;
+                            }
+                        }
+                    }
+                }
+                if *active {
+                    active_local = Some(*local_id);
+                }
+            }
+            Action::Vanished { name, local_id } => {
+                let tx = conn.transaction()?;
+                match apply_delete(&tx, source_id, name, *local_id) {
+                    Ok(()) => {
+                        tx.commit()?;
+                        counts.deleted += 1;
+                    }
+                    Err(e) => {
+                        let _ = tx.rollback();
+                        logger.warn(&format!("DELETE sieve_scripts {name:?} failed: {e}"));
+                        counts.failed += 1;
+                    }
+                }
+            }
+        }
+    }
+
+    apply_active_assignment(&mut conn, active_local)?;
+
+    let _ = client.logout();
+
+    if counts.failed == 0 {
+        let tx = conn.unchecked_transaction()?;
+        db::blobs::gc_orphan_blobs(&tx)?;
+        tx.commit()?;
+    }
+
+    log_at(
+        logger,
+        LEVEL_DEFAULT,
+        &format!(
+            "managesieve: created={} fetched={} deleted={} skipped={} failed={}",
+            counts.created, counts.fetched, counts.deleted, counts.skipped, counts.failed
+        ),
+    );
+
+    Ok(Summary {
+        per_type: vec![("sievescript", counts)],
+        retries_observed: backoff.total_retries(),
+        retry_after_sleeps: backoff.transient_retries() as u64,
+    })
+}
+
+fn build_dry_run_summary(
+    conn: &rusqlite::Connection,
+    source_id: i64,
+    server: &[(String, bool, [u8; 32], Vec<u8>)],
+    mut counts: TypeCounts,
+    backoff: &BackoffState,
+) -> Result<Summary, Error> {
+    let server_keyed: Vec<(String, bool, [u8; 32])> = server
+        .iter()
+        .map(|(n, a, h, _)| (n.clone(), *a, *h))
+        .collect();
+    let (local_map, local_hashes, local_active) = if source_id < 0 {
+        (
+            std::collections::HashMap::new(),
+            std::collections::HashMap::new(),
+            std::collections::HashMap::new(),
+        )
+    } else {
+        let local_map = db::managesieve_ids::all_names(conn, source_id)?;
+        let (h, a) = load_local_state(conn, source_id)?;
+        (local_map, h, a)
+    };
+    let plan = plan(&server_keyed, &local_map, &local_hashes, &local_active);
+    for action in &plan.actions {
+        match action {
+            Action::New { .. } => {
+                counts.created += 1;
+                counts.fetched += 1;
+            }
+            Action::Present {
+                outcome: PresentOutcome::ContentUpdated,
+                ..
+            } => counts.fetched += 1,
+            Action::Present { .. } => counts.skipped += 1,
+            Action::Vanished { .. } => counts.deleted += 1,
+        }
+    }
+    Ok(Summary {
+        per_type: vec![("sievescript", counts)],
+        retries_observed: backoff.total_retries(),
+        retry_after_sleeps: backoff.transient_retries() as u64,
+    })
+}
+
+fn parse_endpoint(url: &str) -> Result<Endpoint, Error> {
+    let parsed =
+        Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?;
+    let scheme = parsed.scheme();
+    let implicit_tls = match scheme {
+        "sieves" => true,
+        "sieve" => false,
+        other => {
+            return Err(Error::Usage(format!(
+                "--url scheme must be sieve or sieves, got {other}"
+            )));
+        }
+    };
+    let host = parsed
+        .host_str()
+        .ok_or_else(|| Error::Usage(format!("--url missing host: {url}")))?
+        .to_owned();
+    let port = parsed.port().unwrap_or(4190);
+    Ok(Endpoint {
+        host,
+        port,
+        implicit_tls,
+    })
+}
+
+fn authenticate(client: &mut SieveClient, auth: &ManageSieveAuth) -> Result<String, Error> {
+    do_authenticate(client, auth).map_err(|e| match e {
+        SieveAuthError::TerminallyRefused(text) => {
+            Error::Connection(format!("auth failed: {text}"))
+        }
+        SieveAuthError::NoUsableMechanism(text) => Error::Connection(text),
+        SieveAuthError::Wire(e) => Error::Connection(e.to_string()),
+    })?;
+    Ok(account_id_for(auth))
+}
+
+fn account_id_for(auth: &ManageSieveAuth) -> String {
+    match auth {
+        ManageSieveAuth::Basic { user, .. } => user.clone(),
+        ManageSieveAuth::Bearer { user, .. } => user.clone(),
+    }
+}
+
+#[derive(Debug)]
+enum SieveAuthError {
+
+    TerminallyRefused(String),
+
+    NoUsableMechanism(String),
+
+    Wire(SieveError),
+}
+
+fn do_authenticate(client: &mut SieveClient, auth: &ManageSieveAuth) -> Result<(), SieveAuthError> {
+    match auth {
+        ManageSieveAuth::Basic { user, password } => {
+            let caps = client.capabilities().clone();
+            let mut last_plain_error: Option<String> = None;
+            if caps.has_sasl("PLAIN") {
+                match client.authenticate_plain(user, password) {
+                    Ok(()) => return Ok(()),
+                    Err(e) if is_negotiation_failure(&e) => {
+                        last_plain_error = Some(e.to_string());
+                    }
+                    Err(e) => return Err(SieveAuthError::Wire(e)),
+                }
+            }
+            if caps.has_sasl("LOGIN") {
+                match client.authenticate_login(user, password) {
+                    Ok(()) => return Ok(()),
+                    Err(e) if is_negotiation_failure(&e) => {
+                        let combined = match last_plain_error {
+                            Some(p) => format!("LOGIN: {e}; PLAIN: {p}"),
+                            None => format!("LOGIN: {e}"),
+                        };
+                        return Err(SieveAuthError::TerminallyRefused(combined));
+                    }
+                    Err(e) => return Err(SieveAuthError::Wire(e)),
+                }
+            }
+            if let Some(p) = last_plain_error {
+                return Err(SieveAuthError::TerminallyRefused(format!("PLAIN: {p}")));
+            }
+            Err(SieveAuthError::NoUsableMechanism(format!(
+                "server SASL list lacks PLAIN and LOGIN (advertised: {:?})",
+                caps.sasl
+            )))
+        }
+        ManageSieveAuth::Bearer { user, token } => {
+            if !client.capabilities().has_sasl("OAUTHBEARER") {
+                return Err(SieveAuthError::NoUsableMechanism(format!(
+                    "server does not advertise OAUTHBEARER (advertised: {:?})",
+                    client.capabilities().sasl
+                )));
+            }
+            match client.authenticate_oauthbearer(user, token) {
+                Ok(()) => Ok(()),
+                Err(e) if is_negotiation_failure(&e) => Err(SieveAuthError::TerminallyRefused(
+                    format!("OAUTHBEARER: {e}"),
+                )),
+                Err(e) => Err(SieveAuthError::Wire(e)),
+            }
+        }
+    }
+}
+
+fn listscripts_with_retry(
+    client: &mut SieveClient,
+    ctx: &ControlCtx,
+) -> Result<Vec<(String, bool)>, Error> {
+    let mut transient_attempts: u32 = 0;
+    let mut transport_attempts: u32 = 0;
+    loop {
+        match client.listscripts() {
+            Ok(block) => {
+                ctx.backoff.reset();
+                let listed = parse_listscripts(&block.data)
+                    .map_err(|e| Error::Connection(format!("LISTSCRIPTS parse: {e}")))?;
+                return Ok(listed.into_iter().map(|s| (s.name, s.active)).collect());
+            }
+            Err(e) => match classify(&e) {
+                Disposition::Transient => {
+                    if transient_attempts >= ctx.max_retries {
+                        return Err(Error::Connection(format!(
+                            "LISTSCRIPTS retries exhausted: {e}"
+                        )));
+                    }
+                    transient_attempts += 1;
+                    log_at(
+                        ctx.logger,
+                        LEVEL_PROGRESS,
+                        &format!("LISTSCRIPTS transient: {e}; retrying"),
+                    );
+                    std::thread::sleep(ctx.backoff.next_shared_delay());
+                }
+                Disposition::TransportDrop => {
+                    if transport_attempts >= ctx.max_retries {
+                        return Err(Error::Connection(format!(
+                            "LISTSCRIPTS transport drop, retries exhausted: {e}"
+                        )));
+                    }
+                    transport_attempts += 1;
+                    std::thread::sleep(ctx.backoff.transport_delay(transport_attempts));
+                    match reconnect(client, ctx) {
+                        Ok(()) => {
+                            log_at(
+                                ctx.logger,
+                                LEVEL_DEFAULT,
+                                &format!("managesieve reconnected after transport drop ({e})"),
+                            );
+                        }
+                        Err(e2) => {
+                            return Err(Error::Connection(format!(
+                                "LISTSCRIPTS reconnect failed: {e2}"
+                            )));
+                        }
+                    }
+                }
+                Disposition::Referral => {
+                    return Err(Error::Connection(format!(
+                        "LISTSCRIPTS referral: {e}; vandelay does not follow referrals"
+                    )));
+                }
+                _ => return Err(Error::Connection(format!("LISTSCRIPTS failed: {e}"))),
+            },
+        }
+    }
+}
+
+fn getscript_with_retry(
+    client: &mut SieveClient,
+    ctx: &ControlCtx,
+    name: &str,
+) -> Result<Vec<u8>, SieveError> {
+    let mut transient_attempts: u32 = 0;
+    let mut transport_attempts: u32 = 0;
+    loop {
+        match client.getscript(name) {
+            Ok(block) => {
+                ctx.backoff.reset();
+                return parse_getscript(&block.data);
+            }
+            Err(e) => match classify(&e) {
+                Disposition::Transient => {
+                    if transient_attempts >= ctx.max_retries {
+                        return Err(e);
+                    }
+                    transient_attempts += 1;
+                    log_at(
+                        ctx.logger,
+                        LEVEL_PROGRESS,
+                        &format!("GETSCRIPT {name:?} transient: {e}; retrying"),
+                    );
+                    std::thread::sleep(ctx.backoff.next_shared_delay());
+                }
+                Disposition::TransportDrop => {
+                    if transport_attempts >= ctx.max_retries {
+                        return Err(e);
+                    }
+                    transport_attempts += 1;
+                    std::thread::sleep(ctx.backoff.transport_delay(transport_attempts));
+                    reconnect(client, ctx)?;
+                    log_at(
+                        ctx.logger,
+                        LEVEL_DEFAULT,
+                        &format!(
+                            "managesieve reconnected mid-GETSCRIPT {name:?} after transport drop"
+                        ),
+                    );
+                }
+                _ => return Err(e),
+            },
+        }
+    }
+}
+
+fn log_at(logger: Logger, level: u8, msg: &str) {
+    if logger.enabled(level) {
+        eprintln!("{msg}");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_endpoint_sieves_defaults_to_4190() {
+        let e = parse_endpoint("sieves://mail.example.com").unwrap();
+        assert_eq!(e.host, "mail.example.com");
+        assert_eq!(e.port, 4190);
+        assert!(e.implicit_tls);
+    }
+
+    #[test]
+    fn parse_endpoint_sieve_defaults_to_4190() {
+        let e = parse_endpoint("sieve://mail.example.com").unwrap();
+        assert_eq!(e.host, "mail.example.com");
+        assert_eq!(e.port, 4190);
+        assert!(!e.implicit_tls);
+    }
+
+    #[test]
+    fn parse_endpoint_explicit_port() {
+        let e = parse_endpoint("sieve://mail.example.com:14190").unwrap();
+        assert_eq!(e.port, 14190);
+        assert!(!e.implicit_tls);
+    }
+
+    #[test]
+    fn parse_endpoint_rejects_other_schemes() {
+        let err = parse_endpoint("imap://example.com").unwrap_err();
+        assert!(matches!(err, Error::Usage(_)));
+    }
+
+    #[test]
+    fn parse_endpoint_rejects_missing_host() {
+        let err = parse_endpoint("sieve:///path").unwrap_err();
+        assert!(matches!(err, Error::Usage(_)));
+    }
+}

src/sync/import_managesieve/reconcile.rs

@@ -0,0 +1,480 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+
+use rusqlite::{Connection, OptionalExtension, params};
+
+use crate::db;
+use crate::error::Error;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PresentOutcome {
+    Unchanged,
+    ActiveOnly,
+    ContentUpdated,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Action {
+    New {
+        name: String,
+        active: bool,
+    },
+    Present {
+        name: String,
+        local_id: i64,
+        outcome: PresentOutcome,
+        active: bool,
+    },
+    Vanished {
+        name: String,
+        local_id: i64,
+    },
+}
+
+pub struct Plan {
+    pub actions: Vec<Action>,
+}
+
+pub fn plan(
+    server: &[(String, bool, [u8; 32])],
+    local: &HashMap<String, i64>,
+    local_hashes: &HashMap<i64, [u8; 32]>,
+    local_active: &HashMap<i64, bool>,
+) -> Plan {
+    let mut actions = Vec::new();
+    let mut server_names: HashSet<&str> = HashSet::new();
+    for (name, active, hash) in server {
+        server_names.insert(name.as_str());
+        match local.get(name) {
+            None => actions.push(Action::New {
+                name: name.clone(),
+                active: *active,
+            }),
+            Some(local_id) => {
+                let cur_hash = local_hashes.get(local_id);
+                let cur_active = local_active.get(local_id).copied().unwrap_or(false);
+                let outcome = if cur_hash != Some(hash) {
+                    PresentOutcome::ContentUpdated
+                } else if cur_active != *active {
+                    PresentOutcome::ActiveOnly
+                } else {
+                    PresentOutcome::Unchanged
+                };
+                actions.push(Action::Present {
+                    name: name.clone(),
+                    local_id: *local_id,
+                    outcome,
+                    active: *active,
+                });
+            }
+        }
+    }
+    for (name, local_id) in local {
+        if !server_names.contains(name.as_str()) {
+            actions.push(Action::Vanished {
+                name: name.clone(),
+                local_id: *local_id,
+            });
+        }
+    }
+    Plan { actions }
+}
+
+pub type LocalState = (HashMap<i64, [u8; 32]>, HashMap<i64, bool>);
+
+pub fn load_local_state(conn: &Connection, source_id: i64) -> Result<LocalState, Error> {
+    let mut stmt = conn.prepare(
+        "SELECT s.id, b.hash, s.is_active
+         FROM sieve_scripts s JOIN blobs b ON b.id = s.blob_id
+         JOIN sync_id_managesieve m ON m.local_id = s.id
+         WHERE m.source_id = ?1",
+    )?;
+    let rows = stmt.query_map(params![source_id], |row| {
+        let id: i64 = row.get(0)?;
+        let hash: Vec<u8> = row.get(1)?;
+        let active: i64 = row.get(2)?;
+        Ok((id, hash, active != 0))
+    })?;
+    let mut hashes = HashMap::new();
+    let mut active = HashMap::new();
+    for r in rows {
+        let (id, hash, a) = r?;
+        if hash.len() == 32 {
+            let mut arr = [0u8; 32];
+            arr.copy_from_slice(&hash);
+            hashes.insert(id, arr);
+        }
+        active.insert(id, a);
+    }
+    Ok((hashes, active))
+}
+
+pub fn apply_new(
+    tx: &rusqlite::Transaction<'_>,
+    source_id: i64,
+    name: &str,
+    bytes: &[u8],
+) -> Result<i64, Error> {
+    let blob_id = db::blobs::intern_blob(tx, bytes)?;
+    tx.execute(
+        "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, 0, ?2)",
+        params![name, blob_id],
+    )?;
+    let local_id = tx.last_insert_rowid();
+    db::managesieve_ids::insert(tx, source_id, name, local_id)?;
+    Ok(local_id)
+}
+
+pub fn apply_content_update(
+    tx: &rusqlite::Transaction<'_>,
+    local_id: i64,
+    bytes: &[u8],
+) -> Result<(), Error> {
+    let blob_id = db::blobs::intern_blob(tx, bytes)?;
+    tx.execute(
+        "UPDATE sieve_scripts SET blob_id = ?1 WHERE id = ?2",
+        params![blob_id, local_id],
+    )?;
+    Ok(())
+}
+
+pub fn apply_delete(
+    tx: &rusqlite::Transaction<'_>,
+    source_id: i64,
+    name: &str,
+    local_id: i64,
+) -> Result<(), Error> {
+    tx.execute("DELETE FROM sieve_scripts WHERE id = ?1", params![local_id])?;
+    db::managesieve_ids::delete(tx, source_id, name)?;
+    Ok(())
+}
+
+pub fn apply_active_assignment(
+    conn: &mut Connection,
+    new_active_local: Option<i64>,
+) -> Result<(), Error> {
+    let tx = conn.transaction()?;
+    let existing_active: Option<i64> = tx
+        .query_row(
+            "SELECT id FROM sieve_scripts WHERE is_active = 1",
+            [],
+            |r| r.get(0),
+        )
+        .optional()?;
+    if existing_active != new_active_local {
+        if existing_active.is_some() {
+            tx.execute(
+                "UPDATE sieve_scripts SET is_active = 0 WHERE is_active = 1",
+                [],
+            )?;
+        }
+        if let Some(id) = new_active_local {
+            tx.execute(
+                "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1",
+                params![id],
+            )?;
+        }
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::db::init;
+    use crate::db::sources::{SourceKey, upsert_source};
+    use blake3::Hasher;
+
+    fn hash(b: &[u8]) -> [u8; 32] {
+        let mut h = Hasher::new();
+        h.update(b);
+        h.finalize().into()
+    }
+
+    fn mem() -> (Connection, i64) {
+        let c = Connection::open_in_memory().unwrap();
+        init::apply_schema(&c).unwrap();
+        let sid = upsert_source(
+            &c,
+            &SourceKey {
+                kind: "managesieve".to_owned(),
+                session_url: "sieve://h:4190".to_owned(),
+                account_id: "a".to_owned(),
+            },
+            None,
+            "a",
+        )
+        .unwrap();
+        (c, sid)
+    }
+
+    #[test]
+    fn plan_classifies_new_present_unchanged_and_vanished() {
+        let h_v = hash(b"v");
+        let h_w = hash(b"w");
+        let server: Vec<(String, bool, [u8; 32])> = vec![
+            ("a".into(), false, h_v),
+            ("b".into(), true, h_w),
+            ("c".into(), false, h_v),
+        ];
+        let mut local: HashMap<String, i64> = HashMap::new();
+        local.insert("b".into(), 2);
+        local.insert("c".into(), 3);
+        local.insert("d".into(), 4);
+        let mut lh: HashMap<i64, [u8; 32]> = HashMap::new();
+        lh.insert(2, h_w);
+        lh.insert(3, h_v);
+        lh.insert(4, h_v);
+        let mut la: HashMap<i64, bool> = HashMap::new();
+        la.insert(2, true);
+        la.insert(3, false);
+        la.insert(4, false);
+        let p = plan(&server, &local, &lh, &la);
+        let mut new_count = 0;
+        let mut unchanged = 0;
+        let mut content_updated = 0;
+        let mut active_only = 0;
+        let mut vanished = 0;
+        for a in p.actions {
+            match a {
+                Action::New { .. } => new_count += 1,
+                Action::Present {
+                    outcome: PresentOutcome::Unchanged,
+                    ..
+                } => unchanged += 1,
+                Action::Present {
+                    outcome: PresentOutcome::ContentUpdated,
+                    ..
+                } => content_updated += 1,
+                Action::Present {
+                    outcome: PresentOutcome::ActiveOnly,
+                    ..
+                } => active_only += 1,
+                Action::Vanished { .. } => vanished += 1,
+            }
+        }
+        assert_eq!(new_count, 1);
+        assert_eq!(unchanged, 2);
+        assert_eq!(content_updated, 0);
+        assert_eq!(active_only, 0);
+        assert_eq!(vanished, 1);
+    }
+
+    #[test]
+    fn plan_flags_content_update_when_hash_differs() {
+        let h_v = hash(b"v");
+        let h_w = hash(b"w");
+        let server: Vec<(String, bool, [u8; 32])> = vec![("a".into(), false, h_w)];
+        let mut local: HashMap<String, i64> = HashMap::new();
+        local.insert("a".into(), 1);
+        let mut lh = HashMap::new();
+        lh.insert(1, h_v);
+        let mut la = HashMap::new();
+        la.insert(1, false);
+        let p = plan(&server, &local, &lh, &la);
+        assert!(matches!(
+            p.actions[0],
+            Action::Present {
+                outcome: PresentOutcome::ContentUpdated,
+                ..
+            }
+        ));
+    }
+
+    #[test]
+    fn plan_flags_active_only_when_only_flag_differs() {
+        let h = hash(b"same");
+        let server: Vec<(String, bool, [u8; 32])> = vec![("a".into(), true, h)];
+        let mut local: HashMap<String, i64> = HashMap::new();
+        local.insert("a".into(), 1);
+        let mut lh = HashMap::new();
+        lh.insert(1, h);
+        let mut la = HashMap::new();
+        la.insert(1, false);
+        let p = plan(&server, &local, &lh, &la);
+        assert!(matches!(
+            p.actions[0],
+            Action::Present {
+                outcome: PresentOutcome::ActiveOnly,
+                ..
+            }
+        ));
+    }
+
+    #[test]
+    fn apply_new_inserts_script_and_id_mapping() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id = apply_new(&tx, sid, "vacation", b"require[\"x\"];").unwrap();
+        tx.commit().unwrap();
+        assert!(id > 0);
+        let local = db::managesieve_ids::local_for(&c, sid, "vacation")
+            .unwrap()
+            .unwrap();
+        assert_eq!(local, id);
+        let count: i64 = c
+            .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(count, 1);
+    }
+
+    #[test]
+    fn apply_content_update_swaps_blob_pointer() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id = apply_new(&tx, sid, "vacation", b"v1").unwrap();
+        tx.commit().unwrap();
+        let tx = c.transaction().unwrap();
+        apply_content_update(&tx, id, b"v2").unwrap();
+        tx.commit().unwrap();
+        let blob: Vec<u8> = c
+            .query_row(
+                "SELECT b.data FROM sieve_scripts s JOIN blobs b ON b.id=s.blob_id WHERE s.id=?1",
+                params![id],
+                |r| r.get(0),
+            )
+            .unwrap();
+        assert_eq!(blob, b"v2");
+    }
+
+    #[test]
+    fn apply_active_assignment_swaps_active_in_one_transaction() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id_a = apply_new(&tx, sid, "a", b"x").unwrap();
+        let id_b = apply_new(&tx, sid, "b", b"y").unwrap();
+        tx.commit().unwrap();
+        apply_active_assignment(&mut c, Some(id_a)).unwrap();
+        let cur: Option<i64> = c
+            .query_row(
+                "SELECT id FROM sieve_scripts WHERE is_active = 1",
+                [],
+                |r| r.get(0),
+            )
+            .optional()
+            .unwrap();
+        assert_eq!(cur, Some(id_a));
+        apply_active_assignment(&mut c, Some(id_b)).unwrap();
+        let cur: Option<i64> = c
+            .query_row(
+                "SELECT id FROM sieve_scripts WHERE is_active = 1",
+                [],
+                |r| r.get(0),
+            )
+            .optional()
+            .unwrap();
+        assert_eq!(cur, Some(id_b));
+        apply_active_assignment(&mut c, None).unwrap();
+        let cur: Option<i64> = c
+            .query_row(
+                "SELECT id FROM sieve_scripts WHERE is_active = 1",
+                [],
+                |r| r.get(0),
+            )
+            .optional()
+            .unwrap();
+        assert_eq!(cur, None);
+    }
+
+    #[test]
+    fn apply_delete_clears_active_before_destroy() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id = apply_new(&tx, sid, "a", b"x").unwrap();
+        tx.commit().unwrap();
+        apply_active_assignment(&mut c, Some(id)).unwrap();
+        let tx = c.transaction().unwrap();
+        apply_delete(&tx, sid, "a", id).unwrap();
+        tx.commit().unwrap();
+        let count: i64 = c
+            .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(count, 0);
+        assert!(
+            db::managesieve_ids::local_for(&c, sid, "a")
+                .unwrap()
+                .is_none()
+        );
+    }
+
+    #[test]
+    fn apply_new_dedups_identical_script_bytes_at_blob_level() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id_a = apply_new(&tx, sid, "a", b"require [\"fileinto\"];\n").unwrap();
+        let id_b = apply_new(&tx, sid, "b", b"require [\"fileinto\"];\n").unwrap();
+        tx.commit().unwrap();
+        let blob_a: i64 = c
+            .query_row(
+                "SELECT blob_id FROM sieve_scripts WHERE id = ?1",
+                params![id_a],
+                |r| r.get(0),
+            )
+            .unwrap();
+        let blob_b: i64 = c
+            .query_row(
+                "SELECT blob_id FROM sieve_scripts WHERE id = ?1",
+                params![id_b],
+                |r| r.get(0),
+            )
+            .unwrap();
+        assert_eq!(blob_a, blob_b, "identical bytes must share a blob row");
+        let blob_count: i64 = c
+            .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(blob_count, 1, "blob table must contain a single row");
+    }
+
+    #[test]
+    fn apply_content_update_orphans_old_blob_for_gc() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id = apply_new(&tx, sid, "a", b"v1").unwrap();
+        tx.commit().unwrap();
+        let tx = c.transaction().unwrap();
+        apply_content_update(&tx, id, b"v2").unwrap();
+        tx.commit().unwrap();
+        let blob_count_before_gc: i64 = c
+            .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(blob_count_before_gc, 2, "old blob still present pre-GC");
+        let tx = c.unchecked_transaction().unwrap();
+        crate::db::blobs::gc_orphan_blobs(&tx).unwrap();
+        tx.commit().unwrap();
+        let blob_count_after_gc: i64 = c
+            .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0))
+            .unwrap();
+        assert_eq!(blob_count_after_gc, 1, "orphan blob reclaimed by GC");
+    }
+
+    #[test]
+    fn partial_unique_index_rejects_two_active_rows() {
+        let (mut c, sid) = mem();
+        let tx = c.transaction().unwrap();
+        let id_a = apply_new(&tx, sid, "a", b"x").unwrap();
+        let id_b = apply_new(&tx, sid, "b", b"y").unwrap();
+        tx.commit().unwrap();
+        c.execute(
+            "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1",
+            params![id_a],
+        )
+        .unwrap();
+        let err = c
+            .execute(
+                "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1",
+                params![id_b],
+            )
+            .unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains("UNIQUE") || msg.contains("unique"),
+            "expected unique-constraint failure, got {msg}"
+        );
+    }
+}

src/sync/import_takeout.rs

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod calendar;
+pub mod contacts;
+pub mod coordinator;
+pub mod labels;
+pub mod mail;
+pub mod mbox;
+pub mod tree;
+pub mod walk;
+
+pub use coordinator::{TakeoutImportConfig, run};

src/sync/import_takeout/calendar.rs

@@ -0,0 +1,309 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+use std::path::Path;
+
+use rusqlite::{Connection, OptionalExtension, Transaction, params};
+use serde_json::Value;
+
+use crate::db::takeout_ids;
+use crate::error::Error;
+use crate::logging::Logger;
+use crate::sync::TypeCounts;
+use crate::sync::import_dav::calcard;
+
+const DEFAULT_CALENDAR: &str = "Imported";
+
+pub fn process_file(
+    conn: &mut Connection,
+    file_path: &Path,
+    source_id: i64,
+    calendar_cache: &mut HashMap<String, i64>,
+    calendar_counts: &mut TypeCounts,
+    event_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let text = match std::fs::read_to_string(file_path) {
+        Ok(s) => s,
+        Err(e) => {
+            logger.warn(&format!("{file_path:?}: read: {e}"));
+            event_counts.failed += 1;
+            return Ok(());
+        }
+    };
+
+    let entries = match calcard::ical_to_jscalendar_entries(&text) {
+        Ok(v) => v,
+        Err(calcard::CalcardError::NoEntries) => {
+            logger.warn(&format!("{file_path:?}: iCalendar contains no events"));
+            return Ok(());
+        }
+        Err(e) => {
+            logger.warn(&format!("{file_path:?}: iCalendar parse: {e}"));
+            event_counts.failed += 1;
+            return Ok(());
+        }
+    };
+
+    let metadata = parse_calendar_metadata(&text);
+    let container_name = metadata
+        .x_wr_calname
+        .clone()
+        .unwrap_or_else(|| DEFAULT_CALENDAR.to_owned());
+
+    let tx = conn.transaction()?;
+    let calendar_local = ensure_calendar(
+        &tx,
+        source_id,
+        &container_name,
+        &metadata,
+        calendar_cache,
+        calendar_counts,
+    )?;
+
+    for (ordinal, mut entry) in entries.into_iter().enumerate() {
+        match insert_event(
+            &tx,
+            source_id,
+            calendar_local,
+            file_path,
+            ordinal,
+            &mut entry,
+            event_counts,
+        ) {
+            Ok(()) => {}
+            Err(e) => {
+                logger.warn(&format!("{file_path:?}#{ordinal}: event insert: {e}"));
+                event_counts.failed += 1;
+            }
+        }
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+#[derive(Debug, Default, Clone)]
+struct CalendarMetadata {
+    x_wr_calname: Option<String>,
+    x_wr_caldesc: Option<String>,
+    x_wr_timezone: Option<String>,
+}
+
+fn ensure_calendar(
+    tx: &Transaction<'_>,
+    source_id: i64,
+    name: &str,
+    metadata: &CalendarMetadata,
+    cache: &mut HashMap<String, i64>,
+    counts: &mut TypeCounts,
+) -> Result<i64, Error> {
+    if let Some(&id) = cache.get(name) {
+        return Ok(id);
+    }
+    if let Some(id) = takeout_ids::local_for(tx, source_id, takeout_ids::CALENDAR, name)? {
+        cache.insert(name.to_owned(), id);
+        return Ok(id);
+    }
+    tx.execute(
+        "INSERT INTO calendars
+           (name, description, time_zone, sort_order, is_subscribed, is_visible, is_default)
+         VALUES (?1, ?2, ?3, 0, 1, 1, 0)",
+        params![name, metadata.x_wr_caldesc, metadata.x_wr_timezone],
+    )?;
+    let id = tx.last_insert_rowid();
+    takeout_ids::insert(tx, source_id, takeout_ids::CALENDAR, name, id)?;
+    cache.insert(name.to_owned(), id);
+    counts.created += 1;
+    counts.fetched += 1;
+    Ok(id)
+}
+
+fn insert_event(
+    tx: &Transaction<'_>,
+    source_id: i64,
+    calendar_local: i64,
+    file_path: &Path,
+    ordinal: usize,
+    entry: &mut calcard::JsCalendarEntry,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let (is_draft, use_default_alerts, uid_field) =
+        calcard::strip_extracted_fields_from_event(&mut entry.data);
+    let uid = match uid_field
+        .as_deref()
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+    {
+        Some(u) => u.to_owned(),
+        None => calcard::synthesise_uid(&format!("{}#{ordinal}", file_path.to_string_lossy())),
+    };
+    let data_type = entry.data_type.as_column();
+    let data_json = entry.data.to_string();
+
+    let existing = takeout_ids::local_for(tx, source_id, takeout_ids::CALENDAR_EVENT, &uid)?;
+
+    if let Some(local) = existing {
+        let calendar_ids =
+            merge_container_id(tx, "calendar_events", "calendar_ids", local, calendar_local)?;
+        let current: (String, i64, i64, String, String) = tx.query_row(
+            "SELECT calendar_ids, is_draft, use_default_alerts, data, data_type
+             FROM calendar_events WHERE id = ?1",
+            params![local],
+            |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
+        )?;
+        let want = (
+            calendar_ids.clone(),
+            is_draft as i64,
+            use_default_alerts as i64,
+            data_json.clone(),
+            data_type.to_owned(),
+        );
+        if current == want {
+            counts.skipped += 1;
+            return Ok(());
+        }
+        tx.execute(
+            "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2,
+                                          use_default_alerts = ?3, data = ?4, data_type = ?5
+             WHERE id = ?6",
+            params![
+                calendar_ids,
+                is_draft as i64,
+                use_default_alerts as i64,
+                data_json,
+                data_type,
+                local,
+            ],
+        )?;
+        counts.fetched += 1;
+        return Ok(());
+    }
+
+    let calendar_ids = format!("[{calendar_local}]");
+    tx.execute(
+        "INSERT INTO calendar_events
+            (calendar_ids, is_draft, use_default_alerts, data, data_type)
+         VALUES (?1, ?2, ?3, ?4, ?5)",
+        params![
+            calendar_ids,
+            is_draft as i64,
+            use_default_alerts as i64,
+            data_json,
+            data_type,
+        ],
+    )?;
+    let local_id = tx.last_insert_rowid();
+    takeout_ids::insert(tx, source_id, takeout_ids::CALENDAR_EVENT, &uid, local_id)?;
+    counts.created += 1;
+    counts.fetched += 1;
+    Ok(())
+}
+
+fn merge_container_id(
+    tx: &Transaction<'_>,
+    table: &str,
+    column: &str,
+    local_id: i64,
+    new_id: i64,
+) -> Result<String, Error> {
+    let current: String = tx
+        .query_row(
+            &format!("SELECT {column} FROM {table} WHERE id = ?1"),
+            params![local_id],
+            |r| r.get(0),
+        )
+        .optional()?
+        .unwrap_or_else(|| "[]".to_owned());
+    let mut ids: Vec<i64> = match serde_json::from_str::<Value>(&current) {
+        Ok(Value::Array(arr)) => arr.into_iter().filter_map(|v| v.as_i64()).collect(),
+        _ => Vec::new(),
+    };
+    if !ids.contains(&new_id) {
+        ids.push(new_id);
+    }
+    ids.sort_unstable();
+    ids.dedup();
+    Ok(Value::Array(ids.into_iter().map(Value::from).collect()).to_string())
+}
+
+fn parse_calendar_metadata(text: &str) -> CalendarMetadata {
+    let mut out = CalendarMetadata::default();
+    let mut inside_vevent = false;
+    for raw_line in text.lines() {
+        let line = raw_line.trim_end_matches('\r');
+        if line.starts_with("BEGIN:VEVENT")
+            || line.starts_with("BEGIN:VTODO")
+            || line.starts_with("BEGIN:VJOURNAL")
+        {
+            inside_vevent = true;
+            continue;
+        }
+        if line.starts_with("END:VEVENT")
+            || line.starts_with("END:VTODO")
+            || line.starts_with("END:VJOURNAL")
+        {
+            inside_vevent = false;
+            continue;
+        }
+        if inside_vevent {
+            continue;
+        }
+        if let Some(rest) = line.strip_prefix("X-WR-CALNAME:") {
+            out.x_wr_calname = Some(rest.trim().to_owned());
+        } else if let Some(rest) = line.strip_prefix("X-WR-CALDESC:") {
+            out.x_wr_caldesc = Some(rest.trim().to_owned());
+        } else if let Some(rest) = line.strip_prefix("X-WR-TIMEZONE:") {
+            out.x_wr_timezone = Some(rest.trim().to_owned());
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_metadata_finds_xwr_fields() {
+        let ical = "BEGIN:VCALENDAR\r\n\
+            PRODID:-//Google Inc//Google Calendar 70.9054//EN\r\n\
+            X-WR-CALNAME:My Calendar\r\n\
+            X-WR-CALDESC:Some description\r\n\
+            X-WR-TIMEZONE:Europe/Madrid\r\n\
+            BEGIN:VEVENT\r\nUID:[email protected]\r\nDTSTAMP:20250101T000000Z\r\n\
+            END:VEVENT\r\nEND:VCALENDAR\r\n";
+        let m = parse_calendar_metadata(ical);
+        assert_eq!(m.x_wr_calname.as_deref(), Some("My Calendar"));
+        assert_eq!(m.x_wr_caldesc.as_deref(), Some("Some description"));
+        assert_eq!(m.x_wr_timezone.as_deref(), Some("Europe/Madrid"));
+    }
+
+    #[test]
+    fn parse_metadata_ignores_xwr_inside_vevent() {
+        let ical = "BEGIN:VCALENDAR\r\n\
+            BEGIN:VEVENT\r\nUID:e1\r\nX-WR-CALNAME:NOT THIS\r\nEND:VEVENT\r\n\
+            END:VCALENDAR\r\n";
+        let m = parse_calendar_metadata(ical);
+        assert!(m.x_wr_calname.is_none());
+    }
+
+    #[test]
+    fn parse_metadata_empty_when_missing() {
+        let ical = "BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n";
+        let m = parse_calendar_metadata(ical);
+        assert!(m.x_wr_calname.is_none());
+        assert!(m.x_wr_caldesc.is_none());
+        assert!(m.x_wr_timezone.is_none());
+    }
+
+    #[test]
+    fn parse_metadata_handles_lf_only_line_endings() {
+        let ical = "BEGIN:VCALENDAR\nX-WR-CALNAME:Foo\nEND:VCALENDAR\n";
+        let m = parse_calendar_metadata(ical);
+        assert_eq!(m.x_wr_calname.as_deref(), Some("Foo"));
+    }
+}

src/sync/import_takeout/contacts.rs

@@ -0,0 +1,264 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::path::Path;
+
+use rusqlite::{Connection, Transaction, params};
+
+use crate::db::takeout_ids;
+use crate::error::Error;
+use crate::logging::Logger;
+use crate::sync::TypeCounts;
+use crate::sync::import_dav::calcard;
+
+const DEFAULT_ADDRESS_BOOK: &str = "Imported";
+
+pub fn process_file(
+    conn: &mut Connection,
+    file_path: &Path,
+    source_id: i64,
+    address_book_local: &mut Option<i64>,
+    book_counts: &mut TypeCounts,
+    card_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let text = match std::fs::read_to_string(file_path) {
+        Ok(s) => s,
+        Err(e) => {
+            logger.warn(&format!("{file_path:?}: read: {e}"));
+            card_counts.failed += 1;
+            return Ok(());
+        }
+    };
+
+    let chunks = split_vcards(&text);
+    if chunks.is_empty() {
+        return Ok(());
+    }
+
+    let tx = conn.transaction()?;
+    let book_local = match *address_book_local {
+        Some(id) => id,
+        None => {
+            let id = ensure_default_address_book(&tx, source_id, book_counts)?;
+            *address_book_local = Some(id);
+            id
+        }
+    };
+
+    for (ordinal, chunk) in chunks.into_iter().enumerate() {
+        match insert_card(
+            &tx,
+            source_id,
+            book_local,
+            file_path,
+            ordinal,
+            &chunk,
+            card_counts,
+        ) {
+            Ok(()) => {}
+            Err(e) => {
+                logger.warn(&format!("{file_path:?}#{ordinal}: vCard: {e}"));
+                card_counts.failed += 1;
+            }
+        }
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+fn ensure_default_address_book(
+    tx: &Transaction<'_>,
+    source_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<i64, Error> {
+    if let Some(id) = takeout_ids::local_for(
+        tx,
+        source_id,
+        takeout_ids::ADDRESS_BOOK,
+        DEFAULT_ADDRESS_BOOK,
+    )? {
+        return Ok(id);
+    }
+    tx.execute(
+        "INSERT INTO address_books
+            (name, description, sort_order, is_default, is_subscribed)
+         VALUES (?1, NULL, 0, 0, 1)",
+        params![DEFAULT_ADDRESS_BOOK],
+    )?;
+    let id = tx.last_insert_rowid();
+    takeout_ids::insert(
+        tx,
+        source_id,
+        takeout_ids::ADDRESS_BOOK,
+        DEFAULT_ADDRESS_BOOK,
+        id,
+    )?;
+    counts.created += 1;
+    counts.fetched += 1;
+    Ok(id)
+}
+
+fn insert_card(
+    tx: &Transaction<'_>,
+    source_id: i64,
+    book_local: i64,
+    file_path: &Path,
+    ordinal: usize,
+    chunk: &str,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let synthetic_href = format!("{}#{ordinal}", file_path.to_string_lossy());
+    let card = calcard::vcard_to_jscontact(chunk, &synthetic_href)
+        .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?;
+    let data_json = card.data.to_string();
+    let address_book_ids = format!("[{book_local}]");
+
+    let existing = takeout_ids::local_for(tx, source_id, takeout_ids::CONTACT_CARD, &card.uid)?;
+
+    if let Some(local) = existing {
+        let current: (String, String, String) = tx.query_row(
+            "SELECT uid, address_book_ids, data FROM contact_cards WHERE id = ?1",
+            params![local],
+            |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+        )?;
+        let want = (
+            card.uid.clone(),
+            address_book_ids.clone(),
+            data_json.clone(),
+        );
+        if current == want {
+            counts.skipped += 1;
+            return Ok(());
+        }
+        tx.execute(
+            "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3
+             WHERE id = ?4",
+            params![card.uid, address_book_ids, data_json, local],
+        )?;
+        counts.fetched += 1;
+        return Ok(());
+    }
+
+    tx.execute(
+        "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)",
+        params![card.uid, address_book_ids, data_json],
+    )?;
+    let local_id = tx.last_insert_rowid();
+    takeout_ids::insert(
+        tx,
+        source_id,
+        takeout_ids::CONTACT_CARD,
+        &card.uid,
+        local_id,
+    )?;
+    counts.created += 1;
+    counts.fetched += 1;
+    Ok(())
+}
+
+fn split_vcards(text: &str) -> Vec<String> {
+    let mut chunks: Vec<String> = Vec::new();
+    let mut current: Option<String> = None;
+    for raw_line in text.lines() {
+        let line = raw_line.trim_end_matches('\r');
+        let bytes = line.as_bytes();
+        if starts_with_ignore_ascii_case(bytes, b"BEGIN:VCARD") {
+            current = Some(String::new());
+        }
+        if let Some(buf) = current.as_mut() {
+            buf.push_str(line);
+            buf.push('\n');
+        }
+        if starts_with_ignore_ascii_case(bytes, b"END:VCARD")
+            && let Some(buf) = current.take()
+        {
+            chunks.push(buf);
+        }
+    }
+    chunks
+}
+
+fn starts_with_ignore_ascii_case(haystack: &[u8], needle: &[u8]) -> bool {
+    haystack.len() >= needle.len() && haystack[..needle.len()].eq_ignore_ascii_case(needle)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn splits_single_card() {
+        let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n";
+        let chunks = split_vcards(text);
+        assert_eq!(chunks.len(), 1);
+        assert!(chunks[0].contains("FN:A"));
+    }
+
+    #[test]
+    fn splits_multiple_cards() {
+        let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\
+            BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\
+            BEGIN:VCARD\r\nVERSION:3.0\r\nFN:C\r\nEND:VCARD\r\n";
+        let chunks = split_vcards(text);
+        assert_eq!(chunks.len(), 3);
+        assert!(chunks[0].contains("FN:A"));
+        assert!(chunks[1].contains("FN:B"));
+        assert!(chunks[2].contains("FN:C"));
+    }
+
+    #[test]
+    fn skips_lines_outside_any_card() {
+        let text = "Some preamble garbage\r\n\
+            BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\
+            garbage between cards\r\n\
+            BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\
+            trailing garbage\r\n";
+        let chunks = split_vcards(text);
+        assert_eq!(chunks.len(), 2);
+        assert!(!chunks[0].contains("garbage"));
+    }
+
+    #[test]
+    fn preserves_folded_continuation_lines_inside_a_card() {
+        let text = "BEGIN:VCARD\r\nVERSION:3.0\r\n\
+            ADR:;;1 Infinite Loop\\nCupertino\\, CA 95014\\nUnited States\r\n \
+            ;;;;;\r\nEND:VCARD\r\n";
+        let chunks = split_vcards(text);
+        assert_eq!(chunks.len(), 1);
+        assert!(chunks[0].contains("ADR:"));
+        assert!(chunks[0].contains(";;;;;"));
+    }
+
+    #[test]
+    fn handles_lf_only_line_endings() {
+        let text = "BEGIN:VCARD\nVERSION:3.0\nFN:LF\nEND:VCARD\n";
+        let chunks = split_vcards(text);
+        assert_eq!(chunks.len(), 1);
+        assert!(chunks[0].contains("FN:LF"));
+    }
+
+    #[test]
+    fn empty_input_yields_no_chunks() {
+        assert!(split_vcards("").is_empty());
+        assert!(split_vcards("not a vcard\n").is_empty());
+    }
+
+    #[test]
+    fn unterminated_card_is_dropped() {
+        let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\n";
+        let chunks = split_vcards(text);
+        assert!(chunks.is_empty());
+    }
+
+    #[test]
+    fn case_insensitive_boundary_match() {
+        let text = "begin:vcard\r\nFN:CaseTest\r\nend:vcard\r\n";
+        let chunks = split_vcards(text);
+        assert_eq!(chunks.len(), 1);
+        assert!(chunks[0].contains("FN:CaseTest"));
+    }
+}

src/sync/import_takeout/coordinator.rs

@@ -0,0 +1,382 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use rusqlite::{Connection, params};
+use url::Url;
+
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::db::takeout_ids;
+use crate::error::Error;
+use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+
+use super::calendar;
+use super::contacts;
+use super::labels::MappingOptions;
+use super::mail;
+use super::walk::{self, FileKind};
+
+#[derive(Debug)]
+pub struct TakeoutImportConfig {
+    pub takeout_root: PathBuf,
+    pub allow_source_change: bool,
+    pub automap: bool,
+}
+
+pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) {
+        eprintln!("takeout importer is single-threaded; --threads value will be ignored");
+    }
+
+    let canonical = std::fs::canonicalize(&config.takeout_root)
+        .map_err(|e| Error::Usage(format!("--path {:?}: {e}", config.takeout_root)))?;
+    if !canonical.is_dir() {
+        return Err(Error::Usage(format!("not a directory: {canonical:?}")));
+    }
+
+    let walk_result = walk::walk_with_logger(&canonical, logger)
+        .map_err(|e| Error::Usage(format!("walking {canonical:?}: {e}")))?;
+    if walk_result.is_empty() {
+        return Err(Error::Usage(format!(
+            "{canonical:?}: no .mbox / .ics / .vcf files found under this path"
+        )));
+    }
+
+    if logger.enabled(LEVEL_PROGRESS) {
+        let mbox_n = walk_result.by_kind(FileKind::Mbox).count();
+        let ics_n = walk_result.by_kind(FileKind::Ics).count();
+        let vcf_n = walk_result.by_kind(FileKind::Vcf).count();
+        eprintln!(
+            "takeout discovery: {} .mbox, {} .ics, {} .vcf (io failures: {}, symlink cycles: {})",
+            mbox_n, ics_n, vcf_n, walk_result.io_failures, walk_result.symlink_cycles
+        );
+    }
+
+    let mut conn = db::init::open(&common.archive)?;
+    let session_url = file_url_for(&canonical)?;
+    let account_id = canonical.to_string_lossy().into_owned();
+    let account_name = canonical
+        .file_name()
+        .and_then(|n| n.to_str())
+        .map(str::to_owned);
+
+    if let Some((existing_url, existing_account)) =
+        db::sources::conflicting_source(&conn, "takeout", &session_url, &account_id)?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records takeout source {existing_url} / {existing_account}; \
+             re-run with --allow-source-change or use a fresh archive"
+        )));
+    }
+
+    let source_key = SourceKey {
+        kind: "takeout".to_owned(),
+        session_url: session_url.clone(),
+        account_id: account_id.clone(),
+    };
+    let source_id = if common.dry_run {
+        db::sources::find_source(&conn, &source_key)?.unwrap_or(-1)
+    } else {
+        db::sources::upsert_source(&conn, &source_key, account_name.as_deref(), "")?
+    };
+
+    if common.dry_run {
+        return Ok(dry_run_summary(&conn, source_id, &walk_result, logger));
+    }
+
+    let options = MappingOptions {
+        automap: config.automap,
+    };
+
+    let mut mailbox_counts = TypeCounts::default();
+    let mut email_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut calendar_event_counts = TypeCounts::default();
+    let mut book_counts = TypeCounts::default();
+    let mut card_counts = TypeCounts::default();
+
+    let mut mailbox_cache: HashMap<String, i64> =
+        takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?;
+
+    process_mbox_files(
+        &mut conn,
+        source_id,
+        &walk_result,
+        options,
+        &mut mailbox_cache,
+        &mut mailbox_counts,
+        &mut email_counts,
+        logger,
+    )?;
+
+    process_ics_files(
+        &mut conn,
+        source_id,
+        &walk_result,
+        &mut calendar_counts,
+        &mut calendar_event_counts,
+        logger,
+    )?;
+
+    process_vcf_files(
+        &mut conn,
+        source_id,
+        &walk_result,
+        &mut book_counts,
+        &mut card_counts,
+        logger,
+    )?;
+
+    let no_failures = mailbox_counts.failed == 0
+        && email_counts.failed == 0
+        && calendar_counts.failed == 0
+        && calendar_event_counts.failed == 0
+        && book_counts.failed == 0
+        && card_counts.failed == 0;
+    if no_failures {
+        let tx = conn.unchecked_transaction()?;
+        db::blobs::gc_orphan_blobs(&tx)?;
+        tx.commit()?;
+    }
+
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "takeout: mailbox c={} f={} | email c={} u={} f={} | calendar c={} f={} | \
+             event c={} u={} f={} | addressbook c={} | card c={} u={} f={}",
+            mailbox_counts.created,
+            mailbox_counts.failed,
+            email_counts.created,
+            email_counts.fetched.saturating_sub(email_counts.created),
+            email_counts.failed,
+            calendar_counts.created,
+            calendar_counts.failed,
+            calendar_event_counts.created,
+            calendar_event_counts
+                .fetched
+                .saturating_sub(calendar_event_counts.created),
+            calendar_event_counts.failed,
+            book_counts.created,
+            card_counts.created,
+            card_counts.fetched.saturating_sub(card_counts.created),
+            card_counts.failed,
+        );
+    }
+
+    Ok(Summary {
+        per_type: vec![
+            ("mailbox", mailbox_counts),
+            ("email", email_counts),
+            ("calendar", calendar_counts),
+            ("calendarevent", calendar_event_counts),
+            ("addressbook", book_counts),
+            ("contactcard", card_counts),
+        ],
+        retries_observed: 0,
+        retry_after_sleeps: 0,
+    })
+}
+
+#[allow(clippy::too_many_arguments)]
+fn process_mbox_files(
+    conn: &mut Connection,
+    source_id: i64,
+    walk_result: &walk::WalkResult,
+    options: MappingOptions,
+    mailbox_cache: &mut HashMap<String, i64>,
+    mailbox_counts: &mut TypeCounts,
+    email_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    for file in walk_result.by_kind(FileKind::Mbox) {
+        let fallback = mailbox_name_from_filename(&file.path);
+        if logger.enabled(LEVEL_PROGRESS) {
+            eprintln!(
+                "mbox: processing {:?} (fallback mailbox: {:?})",
+                file.path, fallback
+            );
+        }
+        let ctx = mail::InsertContext {
+            source_id,
+            fallback_mailbox: &fallback,
+            options,
+            mailbox_cache,
+        };
+        match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) {
+            Ok(()) => {}
+            Err(e) => {
+                logger.warn(&format!("mbox {:?}: {e}", file.path));
+                email_counts.failed += 1;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn process_ics_files(
+    conn: &mut Connection,
+    source_id: i64,
+    walk_result: &walk::WalkResult,
+    calendar_counts: &mut TypeCounts,
+    event_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let mut cache: HashMap<String, i64> =
+        takeout_ids::all_for_type(conn, source_id, takeout_ids::CALENDAR)?;
+    for file in walk_result.by_kind(FileKind::Ics) {
+        if logger.enabled(LEVEL_PROGRESS) {
+            eprintln!("ics: processing {:?}", file.path);
+        }
+        if let Err(e) = calendar::process_file(
+            conn,
+            &file.path,
+            source_id,
+            &mut cache,
+            calendar_counts,
+            event_counts,
+            logger,
+        ) {
+            logger.warn(&format!("ics {:?}: {e}", file.path));
+            event_counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn process_vcf_files(
+    conn: &mut Connection,
+    source_id: i64,
+    walk_result: &walk::WalkResult,
+    book_counts: &mut TypeCounts,
+    card_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let mut book_local: Option<i64> =
+        takeout_ids::local_for(conn, source_id, takeout_ids::ADDRESS_BOOK, "Imported")?;
+    for file in walk_result.by_kind(FileKind::Vcf) {
+        if logger.enabled(LEVEL_PROGRESS) {
+            eprintln!("vcf: processing {:?}", file.path);
+        }
+        if let Err(e) = contacts::process_file(
+            conn,
+            &file.path,
+            source_id,
+            &mut book_local,
+            book_counts,
+            card_counts,
+            logger,
+        ) {
+            logger.warn(&format!("vcf {:?}: {e}", file.path));
+            card_counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn mailbox_name_from_filename(path: &Path) -> String {
+    path.file_stem()
+        .and_then(|s| s.to_str())
+        .map(str::to_owned)
+        .unwrap_or_else(|| "Imported".to_owned())
+}
+
+fn file_url_for(p: &Path) -> Result<String, Error> {
+    Url::from_file_path(p)
+        .map(|u| u.to_string())
+        .map_err(|_| Error::Usage(format!("cannot encode takeout path as file:// URL: {p:?}")))
+}
+
+fn dry_run_summary(
+    conn: &Connection,
+    source_id: i64,
+    walk_result: &walk::WalkResult,
+    logger: Logger,
+) -> Summary {
+    let mbox_n = walk_result.by_kind(FileKind::Mbox).count();
+    let ics_n = walk_result.by_kind(FileKind::Ics).count();
+    let vcf_n = walk_result.by_kind(FileKind::Vcf).count();
+    let local_emails = if source_id < 0 {
+        0
+    } else {
+        count_rows(conn, source_id, takeout_ids::EMAIL)
+    };
+    let local_events = if source_id < 0 {
+        0
+    } else {
+        count_rows(conn, source_id, takeout_ids::CALENDAR_EVENT)
+    };
+    let local_cards = if source_id < 0 {
+        0
+    } else {
+        count_rows(conn, source_id, takeout_ids::CONTACT_CARD)
+    };
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!("dry-run: mbox files={mbox_n} ics files={ics_n} vcf files={vcf_n}");
+        eprintln!(
+            "  existing in archive: emails={local_emails} events={local_events} cards={local_cards}"
+        );
+    }
+    Summary {
+        per_type: vec![
+            ("mailbox", TypeCounts::default()),
+            ("email", TypeCounts::default()),
+            ("calendar", TypeCounts::default()),
+            ("calendarevent", TypeCounts::default()),
+            ("addressbook", TypeCounts::default()),
+            ("contactcard", TypeCounts::default()),
+        ],
+        retries_observed: 0,
+        retry_after_sleeps: 0,
+    }
+}
+
+fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 {
+    conn.query_row(
+        "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2",
+        params![source_id, type_name],
+        |row| row.get::<_, i64>(0),
+    )
+    .unwrap_or(0) as u64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn file_url_encodes_unicode_path() {
+        let p = Path::new("/tmp/Año");
+        let u = file_url_for(p).unwrap();
+        assert!(u.starts_with("file://"));
+        assert!(u.contains("A%C3%B1o"));
+    }
+
+    #[test]
+    fn file_url_rejects_relative_path() {
+        let err = file_url_for(Path::new("./relative")).unwrap_err();
+        assert!(matches!(err, Error::Usage(_)));
+    }
+
+    #[test]
+    fn mailbox_name_from_filename_strips_extension() {
+        let p = Path::new("/some/dir/Github.mbox");
+        assert_eq!(mailbox_name_from_filename(p), "Github");
+    }
+
+    #[test]
+    fn mailbox_name_from_filename_handles_dotted_basename() {
+        let p = Path::new("/x/All mail Including Spam and Trash.mbox");
+        assert_eq!(
+            mailbox_name_from_filename(p),
+            "All mail Including Spam and Trash"
+        );
+    }
+}

src/sync/import_takeout/labels.rs

@@ -0,0 +1,302 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::BTreeSet;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct LabelClassification {
+    pub mailboxes: Vec<String>,
+    pub keywords: BTreeSet<String>,
+    pub had_explicit_seen_signal: bool,
+    pub had_unread_signal: bool,
+    pub opened_won_over_unread: bool,
+}
+
+impl LabelClassification {
+    pub fn keywords_sorted(&self) -> Vec<String> {
+        self.keywords.iter().cloned().collect()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.mailboxes.is_empty() && self.keywords.is_empty()
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct MappingOptions {
+    pub automap: bool,
+}
+
+impl Default for MappingOptions {
+    fn default() -> Self {
+        Self { automap: true }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MailboxAssignment {
+    pub canonical_path: String,
+    pub role: Option<&'static str>,
+}
+
+pub fn parse_header(value: &str) -> Vec<String> {
+    value
+        .split(',')
+        .map(str::trim)
+        .filter(|t| !t.is_empty())
+        .map(str::to_owned)
+        .collect()
+}
+
+pub fn classify(tokens: &[String]) -> LabelClassification {
+    let mut mailboxes: Vec<String> = Vec::new();
+    let mut keywords: BTreeSet<String> = BTreeSet::new();
+    let mut had_opened = false;
+    let mut had_unread = false;
+
+    for tok in tokens {
+        if is_category_token(tok) {
+            continue;
+        }
+        match tok.as_str() {
+            "Starred" => {
+                keywords.insert("$flagged".to_owned());
+            }
+            "Important" => {
+                keywords.insert("$important".to_owned());
+            }
+            "Opened" => {
+                had_opened = true;
+                keywords.insert("$seen".to_owned());
+            }
+            "Unread" => {
+                had_unread = true;
+            }
+            other => {
+                let canonical = canonical_label(other);
+                if !mailboxes.iter().any(|m| m == &canonical) {
+                    mailboxes.push(canonical);
+                }
+            }
+        }
+    }
+
+    let opened_won_over_unread = had_opened && had_unread;
+    LabelClassification {
+        mailboxes,
+        keywords,
+        had_explicit_seen_signal: had_opened,
+        had_unread_signal: had_unread,
+        opened_won_over_unread,
+    }
+}
+
+pub fn role_for_mailbox(canonical_path: &str, opts: MappingOptions) -> Option<&'static str> {
+    if !opts.automap {
+        return None;
+    }
+    if canonical_path.contains('/') {
+        return None;
+    }
+    match canonical_path {
+        "Inbox" => Some("inbox"),
+        "Sent" => Some("sent"),
+        "Drafts" => Some("drafts"),
+        "Trash" => Some("trash"),
+        "Spam" => Some("junk"),
+        "Archive" => Some("archive"),
+        _ => None,
+    }
+}
+
+pub fn assignments_for(
+    classification: &LabelClassification,
+    opts: MappingOptions,
+) -> Vec<MailboxAssignment> {
+    classification
+        .mailboxes
+        .iter()
+        .map(|p| MailboxAssignment {
+            canonical_path: p.clone(),
+            role: role_for_mailbox(p, opts),
+        })
+        .collect()
+}
+
+fn is_category_token(t: &str) -> bool {
+    matches!(
+        t,
+        "Category Personal"
+            | "Category Promotions"
+            | "Category Social"
+            | "Category Updates"
+            | "Category Forums"
+    )
+}
+
+fn canonical_label(raw: &str) -> String {
+    match raw {
+        "Archived" => "Archive".to_owned(),
+        other => other.to_owned(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn classify_str(s: &str) -> LabelClassification {
+        classify(&parse_header(s))
+    }
+
+    #[test]
+    fn parse_header_splits_on_comma_and_trims() {
+        let v = parse_header("Inbox, Opened ,Starred,,Important");
+        assert_eq!(v, vec!["Inbox", "Opened", "Starred", "Important"]);
+    }
+
+    #[test]
+    fn parse_header_empty_yields_empty() {
+        assert!(parse_header("").is_empty());
+        assert!(parse_header(",, ,, ").is_empty());
+    }
+
+    #[test]
+    fn inbox_token_becomes_inbox_mailbox_with_role() {
+        let c = classify_str("Inbox");
+        assert_eq!(c.mailboxes, vec!["Inbox".to_owned()]);
+        let a = assignments_for(&c, MappingOptions::default());
+        assert_eq!(a[0].role, Some("inbox"));
+    }
+
+    #[test]
+    fn sent_drafts_trash_spam_archive_map_to_roles() {
+        for (token, role) in [
+            ("Sent", "sent"),
+            ("Drafts", "drafts"),
+            ("Trash", "trash"),
+            ("Spam", "junk"),
+            ("Archived", "archive"),
+        ] {
+            let c = classify_str(token);
+            let a = assignments_for(&c, MappingOptions::default());
+            assert_eq!(a[0].role, Some(role), "token {token}");
+        }
+        let c = classify_str("Archived");
+        assert_eq!(c.mailboxes, vec!["Archive".to_owned()]);
+    }
+
+    #[test]
+    fn starred_important_become_keywords_not_mailboxes() {
+        let c = classify_str("Starred,Important");
+        assert!(c.mailboxes.is_empty());
+        assert!(c.keywords.contains("$flagged"));
+        assert!(c.keywords.contains("$important"));
+    }
+
+    #[test]
+    fn opened_yields_seen_keyword() {
+        let c = classify_str("Inbox,Opened");
+        assert!(c.keywords.contains("$seen"));
+    }
+
+    #[test]
+    fn unread_alone_means_no_seen() {
+        let c = classify_str("Inbox,Unread");
+        assert!(!c.keywords.contains("$seen"));
+        assert!(c.had_unread_signal);
+    }
+
+    #[test]
+    fn opened_plus_unread_marks_collision_and_keeps_seen() {
+        let c = classify_str("Inbox,Opened,Unread");
+        assert!(c.keywords.contains("$seen"));
+        assert!(c.opened_won_over_unread);
+    }
+
+    #[test]
+    fn category_tokens_are_dropped_silently() {
+        let c = classify_str("Inbox,Category Promotions,Category Updates,Opened");
+        assert_eq!(c.mailboxes, vec!["Inbox".to_owned()]);
+        assert!(c.keywords.contains("$seen"));
+    }
+
+    #[test]
+    fn nested_labels_preserve_full_path() {
+        let c = classify_str("Inbox,Label_001/Label_002_under_Label_001");
+        assert!(
+            c.mailboxes
+                .contains(&"Label_001/Label_002_under_Label_001".to_owned())
+        );
+        let a = assignments_for(&c, MappingOptions::default());
+        let nested = a
+            .iter()
+            .find(|m| m.canonical_path == "Label_001/Label_002_under_Label_001")
+            .unwrap();
+        assert_eq!(nested.role, None);
+    }
+
+    #[test]
+    fn custom_labels_get_no_role() {
+        let c = classify_str("Inbox,Github,Newsletter");
+        let a = assignments_for(&c, MappingOptions::default());
+        let github = a.iter().find(|m| m.canonical_path == "Github").unwrap();
+        assert_eq!(github.role, None);
+    }
+
+    #[test]
+    fn realistic_fixture_label_sets_classify_as_expected() {
+        let c = classify_str("Archived,Important,Opened,Category Social,Github");
+        assert!(c.mailboxes.contains(&"Archive".to_owned()));
+        assert!(c.mailboxes.contains(&"Github".to_owned()));
+        assert!(c.keywords.contains("$important"));
+        assert!(c.keywords.contains("$seen"));
+        assert!(!c.mailboxes.iter().any(|m| m.contains("Category")));
+
+        let c = classify_str("Important,Trash,Category Social,Unread,Github");
+        assert!(c.mailboxes.contains(&"Trash".to_owned()));
+        assert!(c.mailboxes.contains(&"Github".to_owned()));
+        assert!(c.keywords.contains("$important"));
+        assert!(!c.keywords.contains("$seen"));
+        assert!(c.had_unread_signal);
+    }
+
+    #[test]
+    fn noautomap_suppresses_roles_but_keeps_names() {
+        let c = classify_str("Inbox,Sent,Github");
+        let a = assignments_for(&c, MappingOptions { automap: false });
+        assert!(a.iter().all(|m| m.role.is_none()));
+        let names: Vec<&str> = a.iter().map(|m| m.canonical_path.as_str()).collect();
+        assert!(names.contains(&"Inbox"));
+        assert!(names.contains(&"Sent"));
+        assert!(names.contains(&"Github"));
+    }
+
+    #[test]
+    fn keywords_sorted_returns_deterministic_order() {
+        let c = classify_str("Starred,Important,Opened");
+        let k = c.keywords_sorted();
+        assert_eq!(
+            k,
+            vec![
+                "$flagged".to_owned(),
+                "$important".to_owned(),
+                "$seen".to_owned()
+            ]
+        );
+    }
+
+    #[test]
+    fn duplicate_mailbox_token_dedups() {
+        let c = classify_str("Github,Github,Inbox");
+        let count = c
+            .mailboxes
+            .iter()
+            .filter(|m| m.as_str() == "Github")
+            .count();
+        assert_eq!(count, 1);
+    }
+}

src/sync/import_takeout/mail.rs

@@ -0,0 +1,416 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::Path;
+use std::time::{Duration, UNIX_EPOCH};
+
+use rusqlite::{Connection, OptionalExtension, Transaction, params};
+use serde_json::Value;
+use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
+
+use crate::db;
+use crate::db::takeout_ids;
+use crate::error::Error;
+use crate::logging::{LEVEL_PROGRESS, Logger};
+use crate::sync::TypeCounts;
+use crate::sync::emailmeta::email_meta_from_blob;
+use crate::sync::keys::index_to_json;
+
+use super::labels::{self, MappingOptions};
+use super::mbox::{Message, MessageIterator};
+
+pub struct InsertContext<'a> {
+    pub source_id: i64,
+    pub fallback_mailbox: &'a str,
+    pub options: MappingOptions,
+    pub mailbox_cache: &'a mut HashMap<String, i64>,
+}
+
+pub fn process_file(
+    conn: &mut Connection,
+    file_path: &Path,
+    ctx: InsertContext<'_>,
+    mailbox_counts: &mut TypeCounts,
+    email_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let file =
+        File::open(file_path).map_err(|e| Error::Partial(format!("open {file_path:?}: {e}")))?;
+    let reader = BufReader::new(file);
+    let mut iter = MessageIterator::new(reader);
+
+    let InsertContext {
+        source_id,
+        fallback_mailbox,
+        options,
+        mailbox_cache,
+    } = ctx;
+    let tx = conn.transaction()?;
+    let mut commit_pending: u64 = 0;
+
+    loop {
+        match iter.next() {
+            None => break,
+            Some(Err(e)) => {
+                logger.warn(&format!("{file_path:?}: mbox io: {e}"));
+                email_counts.failed += 1;
+                break;
+            }
+            Some(Ok(msg)) => {
+                match process_message(
+                    &tx,
+                    source_id,
+                    fallback_mailbox,
+                    options,
+                    &msg,
+                    mailbox_cache,
+                    mailbox_counts,
+                    email_counts,
+                    logger,
+                ) {
+                    Ok(()) => {
+                        commit_pending += 1;
+                        if commit_pending.is_multiple_of(PROGRESS_TICK)
+                            && logger.enabled(LEVEL_PROGRESS)
+                        {
+                            eprintln!("{file_path:?}: processed {commit_pending} messages");
+                        }
+                    }
+                    Err(e) => {
+                        logger.warn(&format!("{file_path:?}: message: {e}"));
+                        email_counts.failed += 1;
+                    }
+                }
+            }
+        }
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+const PROGRESS_TICK: u64 = 256;
+
+#[allow(clippy::too_many_arguments)]
+fn process_message(
+    tx: &Transaction<'_>,
+    source_id: i64,
+    fallback_mailbox: &str,
+    options: MappingOptions,
+    msg: &Message,
+    cache: &mut HashMap<String, i64>,
+    mailbox_counts: &mut TypeCounts,
+    email_counts: &mut TypeCounts,
+    logger: Logger,
+) -> Result<(), Error> {
+    let bytes = msg.contents();
+    let msg_hash = blake3::hash(bytes).to_hex().to_string();
+
+    let raw_header = extract_x_gmail_labels(bytes);
+    let had_labels_header = raw_header.is_some();
+    let tokens = match &raw_header {
+        Some(s) => labels::parse_header(s),
+        None => Vec::new(),
+    };
+    let mut classification = labels::classify(&tokens);
+    if classification.mailboxes.is_empty() {
+        let fallback = fallback_mailbox.trim();
+        if fallback.is_empty() {
+            classification.mailboxes.push("Imported".to_owned());
+        } else {
+            classification.mailboxes.push(fallback.to_owned());
+        }
+        if had_labels_header {
+            logger.warn(&format!(
+                "message {msg_hash}: X-Gmail-Labels present but empty after parse; \
+                 placing in fallback mailbox {:?}",
+                classification.mailboxes[0]
+            ));
+        } else {
+            logger.warn(&format!(
+                "message {msg_hash}: no X-Gmail-Labels header; placing in fallback mailbox {:?}",
+                classification.mailboxes[0]
+            ));
+        }
+    }
+    if classification.opened_won_over_unread {
+        logger.warn(&format!(
+            "message {msg_hash}: X-Gmail-Labels carried both 'Opened' and 'Unread'; \
+             treating as $seen (Opened wins)"
+        ));
+    }
+
+    let mut mailbox_ids: Vec<i64> = Vec::with_capacity(classification.mailboxes.len());
+    for path in &classification.mailboxes {
+        let id = ensure_mailbox(tx, source_id, path, options, cache, mailbox_counts)?;
+        mailbox_ids.push(id);
+    }
+    mailbox_ids.sort_unstable();
+    mailbox_ids.dedup();
+
+    let keywords_json = json_string_array(&classification.keywords_sorted());
+    let mailbox_ids_json = json_int_array(&mailbox_ids);
+
+    if let Some(local_id) = takeout_ids::local_for(tx, source_id, takeout_ids::EMAIL, &msg_hash)? {
+        let current: (String, String) = tx
+            .query_row(
+                "SELECT mailbox_ids, keywords FROM emails WHERE id = ?1",
+                params![local_id],
+                |r| Ok((r.get(0)?, r.get(1)?)),
+            )
+            .optional()?
+            .unwrap_or_default();
+        if current.0 != mailbox_ids_json || current.1 != keywords_json {
+            tx.execute(
+                "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 WHERE id = ?3",
+                params![mailbox_ids_json, keywords_json, local_id],
+            )?;
+            email_counts.fetched += 1;
+        } else {
+            email_counts.skipped += 1;
+        }
+        return Ok(());
+    }
+
+    let blob_id = db::blobs::intern_blob(tx, bytes)?;
+    let (index, date_rfc3339) = email_meta_from_blob(bytes);
+    let message_match = index_to_json(&index);
+    let received_at = pick_received_at(msg.internal_date(), date_rfc3339.as_deref());
+
+    tx.execute(
+        "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match)
+         VALUES (?1, ?2, ?3, ?4, ?5)",
+        params![
+            blob_id,
+            received_at,
+            mailbox_ids_json,
+            keywords_json,
+            message_match
+        ],
+    )?;
+    let local_id = tx.last_insert_rowid();
+    takeout_ids::insert(tx, source_id, takeout_ids::EMAIL, &msg_hash, local_id)?;
+    email_counts.created += 1;
+    email_counts.fetched += 1;
+    Ok(())
+}
+
+fn ensure_mailbox(
+    tx: &Transaction<'_>,
+    source_id: i64,
+    path: &str,
+    options: MappingOptions,
+    cache: &mut HashMap<String, i64>,
+    counts: &mut TypeCounts,
+) -> Result<i64, Error> {
+    if let Some(&id) = cache.get(path) {
+        return Ok(id);
+    }
+    if let Some(id) = takeout_ids::local_for(tx, source_id, takeout_ids::MAILBOX, path)? {
+        cache.insert(path.to_owned(), id);
+        return Ok(id);
+    }
+    let (parent_id, leaf) = match path.rsplit_once('/') {
+        Some((parent, leaf)) => (
+            Some(ensure_mailbox(
+                tx, source_id, parent, options, cache, counts,
+            )?),
+            leaf,
+        ),
+        None => (None, path),
+    };
+    let role = labels::role_for_mailbox(path, options);
+    tx.execute(
+        "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed)
+         VALUES (?1, ?2, ?3, 0, 1)",
+        params![leaf, parent_id, role],
+    )?;
+    let id = tx.last_insert_rowid();
+    takeout_ids::insert(tx, source_id, takeout_ids::MAILBOX, path, id)?;
+    cache.insert(path.to_owned(), id);
+    counts.created += 1;
+    counts.fetched += 1;
+    Ok(id)
+}
+
+fn json_string_array(values: &[String]) -> String {
+    Value::Array(values.iter().map(|s| Value::String(s.clone())).collect()).to_string()
+}
+
+fn json_int_array(values: &[i64]) -> String {
+    Value::Array(values.iter().map(|&i| Value::from(i)).collect()).to_string()
+}
+
+fn pick_received_at(internal_date: u64, date_rfc3339: Option<&str>) -> String {
+    if internal_date > 0 {
+        return format_unix_rfc3339(internal_date);
+    }
+    if let Some(d) = date_rfc3339 {
+        return d.to_owned();
+    }
+    OffsetDateTime::now_utc()
+        .format(&Rfc3339)
+        .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned())
+}
+
+fn format_unix_rfc3339(secs: u64) -> String {
+    let when = UNIX_EPOCH + Duration::from_secs(secs);
+    OffsetDateTime::from(when)
+        .format(&Rfc3339)
+        .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned())
+}
+
+fn extract_x_gmail_labels(bytes: &[u8]) -> Option<String> {
+    let mut values: Vec<String> = Vec::new();
+    let mut current: Option<String> = None;
+    let mut i = 0;
+    while i < bytes.len() {
+        let nl = bytes[i..].iter().position(|&b| b == b'\n');
+        let (line, advance) = match nl {
+            Some(p) => (&bytes[i..i + p], p + 1),
+            None => (&bytes[i..], bytes.len() - i),
+        };
+        let line = line.strip_suffix(b"\r").unwrap_or(line);
+        if line.is_empty() {
+            break;
+        }
+        let is_continuation = line
+            .first()
+            .map(|b| *b == b' ' || *b == b'\t')
+            .unwrap_or(false);
+        if is_continuation {
+            if let Some(buf) = current.as_mut() {
+                let extra = std::str::from_utf8(line).unwrap_or("").trim_start();
+                if !extra.is_empty() {
+                    buf.push(' ');
+                    buf.push_str(extra);
+                }
+            }
+        } else {
+            if let Some(buf) = current.take() {
+                values.push(buf);
+            }
+            if let Some(colon) = line.iter().position(|&b| b == b':') {
+                let name = &line[..colon];
+                if name.eq_ignore_ascii_case(b"X-Gmail-Labels") {
+                    let value = &line[colon + 1..];
+                    let value = std::str::from_utf8(value).unwrap_or("").trim();
+                    current = Some(value.to_owned());
+                }
+            }
+        }
+        i += advance;
+    }
+    if let Some(buf) = current.take() {
+        values.push(buf);
+    }
+    if values.is_empty() {
+        None
+    } else {
+        Some(values.join(","))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn extract_labels_from_simple_message() {
+        let bytes = b"From: a@b\r\n\
+            X-Gmail-Labels: Inbox,Opened\r\n\
+            Subject: hi\r\n\
+            \r\n\
+            body\r\n";
+        assert_eq!(
+            extract_x_gmail_labels(bytes),
+            Some("Inbox,Opened".to_owned())
+        );
+    }
+
+    #[test]
+    fn extract_labels_handles_lf_only_line_endings() {
+        let bytes = b"From: a@b\nX-Gmail-Labels: Inbox\n\nbody\n";
+        assert_eq!(extract_x_gmail_labels(bytes), Some("Inbox".to_owned()));
+    }
+
+    #[test]
+    fn extract_labels_handles_folded_continuation() {
+        let bytes = b"From: a@b\r\n\
+            X-Gmail-Labels: Inbox,\r\n\
+            \tImportant,Github\r\n\
+            Subject: hi\r\n\
+            \r\n\
+            body\r\n";
+        assert_eq!(
+            extract_x_gmail_labels(bytes),
+            Some("Inbox, Important,Github".to_owned())
+        );
+    }
+
+    #[test]
+    fn extract_labels_returns_none_when_header_absent() {
+        let bytes = b"From: a@b\r\nSubject: hi\r\n\r\nbody\r\n";
+        assert_eq!(extract_x_gmail_labels(bytes), None);
+    }
+
+    #[test]
+    fn extract_labels_case_insensitive_name_match() {
+        let bytes = b"x-gmail-labels: Sent\r\n\r\n";
+        assert_eq!(extract_x_gmail_labels(bytes), Some("Sent".to_owned()));
+    }
+
+    #[test]
+    fn extract_labels_concatenates_multiple_header_occurrences() {
+        let bytes = b"X-Gmail-Labels: Inbox\r\nX-Gmail-Labels: Github\r\n\r\nbody";
+        assert_eq!(
+            extract_x_gmail_labels(bytes),
+            Some("Inbox,Github".to_owned())
+        );
+    }
+
+    #[test]
+    fn extract_labels_stops_at_blank_line() {
+        let bytes = b"X-Gmail-Labels: Inbox\r\n\r\nX-Gmail-Labels: NOT-A-HEADER\r\n";
+        assert_eq!(extract_x_gmail_labels(bytes), Some("Inbox".to_owned()));
+    }
+
+    #[test]
+    fn json_int_array_emits_compact_form() {
+        assert_eq!(json_int_array(&[1, 2, 3]), "[1,2,3]");
+        assert_eq!(json_int_array(&[]), "[]");
+    }
+
+    #[test]
+    fn json_string_array_quotes_each_value() {
+        assert_eq!(
+            json_string_array(&["$seen".to_owned(), "$flagged".to_owned()]),
+            "[\"$seen\",\"$flagged\"]"
+        );
+    }
+
+    #[test]
+    fn pick_received_at_prefers_envelope_internal_date() {
+        let got = pick_received_at(1763065233, Some("2025-05-12T10:00:00+02:00"));
+        assert_eq!(got, "2025-11-13T20:20:33Z");
+    }
+
+    #[test]
+    fn pick_received_at_falls_back_to_date_header() {
+        let got = pick_received_at(0, Some("2025-05-12T10:00:00+02:00"));
+        assert_eq!(got, "2025-05-12T10:00:00+02:00");
+    }
+
+    #[test]
+    fn pick_received_at_last_resort_is_now_in_rfc3339() {
+        let got = pick_received_at(0, None);
+        assert!(got.ends_with("Z") || got.contains("+") || got.contains("-"));
+        assert!(got.contains("T"));
+    }
+}

src/sync/import_takeout/mbox.rs

@@ -0,0 +1,278 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::io::BufRead;
+
+use time::{Date, Month, OffsetDateTime, Time, UtcOffset};
+
+pub struct MessageIterator<T> {
+    reader: T,
+    message: Option<Message>,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct Message {
+    internal_date: u64,
+    envelope_from: String,
+    contents: Vec<u8>,
+}
+
+impl<T> MessageIterator<T>
+where
+    T: BufRead,
+{
+    pub fn new(reader: T) -> MessageIterator<T> {
+        MessageIterator {
+            reader,
+            message: None,
+        }
+    }
+}
+
+impl<T> Iterator for MessageIterator<T>
+where
+    T: BufRead,
+{
+    type Item = std::io::Result<Message>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut line = Vec::with_capacity(80);
+
+        loop {
+            match self.reader.read_until(b'\n', &mut line) {
+                Ok(0) => return self.message.take().map(Ok),
+                Ok(_) => {}
+                Err(e) => return Some(Err(e)),
+            }
+
+            if line.starts_with(b"From ") {
+                let finished = self.message.take().map(Ok);
+                let header = std::str::from_utf8(&line).unwrap_or("");
+                self.message = Some(Message::from_envelope(header));
+                if finished.is_some() {
+                    return finished;
+                }
+                line.clear();
+                continue;
+            }
+
+            if let Some(message) = &mut self.message {
+                if line.first() != Some(&b'>') {
+                    message.contents.extend_from_slice(&line);
+                    line.clear();
+                    continue;
+                }
+                let non_gt = line.iter().position(|&c| c != b'>').unwrap_or(line.len());
+                if line[non_gt..].starts_with(b"From ") {
+                    message.contents.extend_from_slice(&line[1..]);
+                } else {
+                    message.contents.extend_from_slice(&line);
+                }
+            }
+            line.clear();
+        }
+    }
+}
+
+impl Message {
+    fn from_envelope(header: &str) -> Self {
+        let trimmed = header.strip_prefix("From ").unwrap_or(header);
+        let (envelope_from, date_str) = match trimmed.split_once(' ') {
+            Some((from, rest)) => (from.trim().to_owned(), rest),
+            None => (String::new(), ""),
+        };
+        let internal_date = parse_envelope_date(date_str).unwrap_or(0);
+        Self {
+            internal_date,
+            envelope_from,
+            contents: Vec::with_capacity(1024),
+        }
+    }
+
+    pub fn internal_date(&self) -> u64 {
+        self.internal_date
+    }
+
+    pub fn envelope_from(&self) -> &str {
+        &self.envelope_from
+    }
+
+    pub fn contents(&self) -> &[u8] {
+        &self.contents
+    }
+
+    pub fn unwrap_contents(self) -> Vec<u8> {
+        self.contents
+    }
+}
+
+fn parse_envelope_date(s: &str) -> Option<u64> {
+    let parts: Vec<&str> = s.split_whitespace().collect();
+    let (month_pos, day_pos, time_pos, offset_pos, year_pos) = match parts.len() {
+        5 => (1, 2, 3, None, 4),
+        6 => (1, 2, 3, Some(4), 5),
+        _ => return None,
+    };
+
+    let month = parse_month(parts[month_pos])?;
+    let day: u8 = parts[day_pos].parse().ok()?;
+    let (hour, minute, second) = parse_hms(parts[time_pos])?;
+    let year: i32 = parts[year_pos].parse().ok()?;
+    let offset = match offset_pos {
+        Some(p) => parse_offset(parts[p])?,
+        None => UtcOffset::UTC,
+    };
+
+    let date = Date::from_calendar_date(year, month, day).ok()?;
+    let time = Time::from_hms(hour, minute, second).ok()?;
+    let ts = OffsetDateTime::new_in_offset(date, time, offset).unix_timestamp();
+    u64::try_from(ts).ok()
+}
+
+fn parse_month(s: &str) -> Option<Month> {
+    Some(match s {
+        x if x.eq_ignore_ascii_case("jan") => Month::January,
+        x if x.eq_ignore_ascii_case("feb") => Month::February,
+        x if x.eq_ignore_ascii_case("mar") => Month::March,
+        x if x.eq_ignore_ascii_case("apr") => Month::April,
+        x if x.eq_ignore_ascii_case("may") => Month::May,
+        x if x.eq_ignore_ascii_case("jun") => Month::June,
+        x if x.eq_ignore_ascii_case("jul") => Month::July,
+        x if x.eq_ignore_ascii_case("aug") => Month::August,
+        x if x.eq_ignore_ascii_case("sep") => Month::September,
+        x if x.eq_ignore_ascii_case("oct") => Month::October,
+        x if x.eq_ignore_ascii_case("nov") => Month::November,
+        x if x.eq_ignore_ascii_case("dec") => Month::December,
+        _ => return None,
+    })
+}
+
+fn parse_hms(s: &str) -> Option<(u8, u8, u8)> {
+    let mut it = s.split(':');
+    let h: u8 = it.next()?.parse().ok()?;
+    let m: u8 = it.next()?.parse().ok()?;
+    let sec: u8 = it.next()?.parse().ok()?;
+    if it.next().is_some() {
+        return None;
+    }
+    Some((h, m, sec))
+}
+
+fn parse_offset(s: &str) -> Option<UtcOffset> {
+    let bytes = s.as_bytes();
+    if bytes.len() != 5 {
+        return None;
+    }
+    let sign: i8 = match bytes[0] {
+        b'+' => 1,
+        b'-' => -1,
+        _ => return None,
+    };
+    let hh: i8 = std::str::from_utf8(&bytes[1..3]).ok()?.parse().ok()?;
+    let mm: i8 = std::str::from_utf8(&bytes[3..5]).ok()?.parse().ok()?;
+    UtcOffset::from_hms(sign * hh, sign * mm, 0).ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn classic_ctime_envelope_parses() {
+        let raw = b"From [email protected] Sat Jan  3 01:05:34 1996\n\
+            Message 1\n\
+            \n\
+            From [email protected]  Tue Jul 23 19:39:23 2002\n\
+            Message 2\n\
+            \n";
+        let mut it = MessageIterator::new(&raw[..]);
+        let m1 = it.next().unwrap().unwrap();
+        assert_eq!(m1.envelope_from(), "[email protected]");
+        assert_eq!(m1.internal_date(), 820631134);
+        assert_eq!(m1.contents(), b"Message 1\n\n");
+        let m2 = it.next().unwrap().unwrap();
+        assert_eq!(m2.envelope_from(), "[email protected]");
+        assert_eq!(m2.internal_date(), 1027453163);
+        assert!(it.next().is_none());
+    }
+
+    #[test]
+    fn mboxrd_unescape_recovers_lines() {
+        let raw = b"From a@b Tue Aug  6 13:34:34 2002\n\
+            Message 3\n\
+            >From hello\n\
+            >>From world\n\
+            >>>From test\n\
+            \n";
+        let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap();
+        assert_eq!(
+            m.contents(),
+            b"Message 3\nFrom hello\n>From world\n>>From test\n\n"
+        );
+    }
+
+    #[test]
+    fn mboxrd_leaves_non_from_lines_starting_with_gt_alone() {
+        let raw = b"From a@b Mon Jan 15  15:30:00  2018\n\
+            Message 4\n\
+            > From\n\
+            >F\n";
+        let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap();
+        assert_eq!(m.contents(), b"Message 4\n> From\n>F\n");
+    }
+
+    #[test]
+    fn takeout_envelope_with_utc_offset_parses() {
+        let raw = b"From 1848707889910060830@xxx Thu Nov 13 20:20:33 +0000 2025\n\
+            X-GM-THRID: 1848707889910060830\n\
+            X-Gmail-Labels: Inbox,Opened\n\
+            Subject: hi\n\
+            \n\
+            body\n";
+        let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap();
+        assert_eq!(m.envelope_from(), "1848707889910060830@xxx");
+        assert_eq!(m.internal_date(), 1763065233);
+        assert!(m.contents().starts_with(b"X-GM-THRID:"));
+    }
+
+    #[test]
+    fn takeout_envelope_with_negative_offset_applies_offset() {
+        let raw = b"From x@y Thu Nov 13 12:20:33 -0800 2025\n\
+            body\n";
+        let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap();
+        assert_eq!(m.internal_date(), 1763065233);
+    }
+
+    #[test]
+    fn malformed_envelope_date_yields_zero_not_error() {
+        let raw = b"From who knows what\nbody\n";
+        let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap();
+        assert_eq!(m.internal_date(), 0);
+    }
+
+    #[test]
+    fn streams_multiple_takeout_format_messages() {
+        let raw = b"From 1@xxx Thu Nov 13 20:20:33 +0000 2025\n\
+            X-GM-THRID: 1\n\
+            X-Gmail-Labels: Inbox\n\
+            Subject: A\n\
+            \n\
+            body a\n\
+            From 2@xxx Thu Nov 13 21:00:00 +0000 2025\n\
+            X-GM-THRID: 2\n\
+            X-Gmail-Labels: Sent,Archived\n\
+            Subject: B\n\
+            \n\
+            body b\n";
+        let parsed: Vec<_> = MessageIterator::new(&raw[..])
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+        assert_eq!(parsed.len(), 2);
+        assert_eq!(parsed[0].envelope_from(), "1@xxx");
+        assert_eq!(parsed[1].envelope_from(), "2@xxx");
+        assert!(parsed[0].contents().starts_with(b"X-GM-THRID: 1"));
+    }
+}

src/sync/import_takeout/tree.rs

@@ -0,0 +1,210 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::BTreeSet;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PlannedMailbox {
+    pub canonical_path: String,
+    pub leaf: String,
+    pub parent_path: Option<String>,
+    pub role: Option<&'static str>,
+    pub ephemeral: bool,
+}
+
+pub fn assemble<F>(input: &[String], role_for: F) -> Vec<PlannedMailbox>
+where
+    F: Fn(&str) -> Option<&'static str>,
+{
+    let explicit: BTreeSet<String> = input.iter().cloned().collect();
+    let mut all_paths: BTreeSet<String> = BTreeSet::new();
+    for p in &explicit {
+        for prefix in prefixes(p) {
+            all_paths.insert(prefix);
+        }
+    }
+
+    let mut planned: Vec<PlannedMailbox> = Vec::with_capacity(all_paths.len());
+    for path in all_paths {
+        let (parent_path, leaf) = split_path(&path);
+        let ephemeral = !explicit.contains(&path);
+        let role = role_for(&path);
+        planned.push(PlannedMailbox {
+            canonical_path: path,
+            leaf,
+            parent_path,
+            role,
+            ephemeral,
+        });
+    }
+    planned
+}
+
+pub fn vanished_depth_sort(paths: &mut [String]) {
+    paths.sort_by(|a, b| depth_of(b).cmp(&depth_of(a)).then_with(|| a.cmp(b)));
+}
+
+fn prefixes(path: &str) -> Vec<String> {
+    let parts: Vec<&str> = path.split('/').collect();
+    let mut out = Vec::with_capacity(parts.len());
+    for i in 1..=parts.len() {
+        out.push(parts[..i].join("/"));
+    }
+    out
+}
+
+fn split_path(path: &str) -> (Option<String>, String) {
+    match path.rsplit_once('/') {
+        Some((parent, leaf)) => (Some(parent.to_owned()), leaf.to_owned()),
+        None => (None, path.to_owned()),
+    }
+}
+
+fn depth_of(path: &str) -> usize {
+    path.bytes().filter(|b| *b == b'/').count() + 1
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn no_roles(_: &str) -> Option<&'static str> {
+        None
+    }
+
+    fn inbox_role(p: &str) -> Option<&'static str> {
+        if p == "Inbox" { Some("inbox") } else { None }
+    }
+
+    #[test]
+    fn flat_paths_pass_through_unchanged() {
+        let r = assemble(
+            &["Inbox".to_owned(), "Sent".to_owned(), "Github".to_owned()],
+            no_roles,
+        );
+        let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect();
+        assert_eq!(names, vec!["Github", "Inbox", "Sent"]);
+        assert!(r.iter().all(|m| !m.ephemeral));
+        assert!(r.iter().all(|m| m.parent_path.is_none()));
+    }
+
+    #[test]
+    fn nested_path_creates_ephemeral_parent() {
+        let r = assemble(
+            &["Label_001/Label_002_under_Label_001".to_owned()],
+            no_roles,
+        );
+        assert_eq!(r.len(), 2);
+        assert_eq!(r[0].canonical_path, "Label_001");
+        assert!(r[0].ephemeral);
+        assert_eq!(r[0].parent_path, None);
+        assert_eq!(r[1].canonical_path, "Label_001/Label_002_under_Label_001");
+        assert!(!r[1].ephemeral);
+        assert_eq!(r[1].parent_path, Some("Label_001".to_owned()));
+        assert_eq!(r[1].leaf, "Label_002_under_Label_001");
+    }
+
+    #[test]
+    fn explicit_parent_is_not_marked_ephemeral() {
+        let r = assemble(
+            &[
+                "Label_001".to_owned(),
+                "Label_001/Sub".to_owned(),
+                "Label_001/Other".to_owned(),
+            ],
+            no_roles,
+        );
+        let parent = r.iter().find(|m| m.canonical_path == "Label_001").unwrap();
+        assert!(!parent.ephemeral);
+    }
+
+    #[test]
+    fn three_level_deep_path_creates_all_missing_ancestors() {
+        let r = assemble(&["A/B/C".to_owned()], no_roles);
+        let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect();
+        assert_eq!(names, vec!["A", "A/B", "A/B/C"]);
+        assert!(r[0].ephemeral);
+        assert!(r[1].ephemeral);
+        assert!(!r[2].ephemeral);
+        assert_eq!(r[2].parent_path, Some("A/B".to_owned()));
+    }
+
+    #[test]
+    fn role_closure_consulted_for_ephemeral_parents_too() {
+        let r = assemble(&["Inbox/Sub".to_owned()], inbox_role);
+        let inbox = r.iter().find(|m| m.canonical_path == "Inbox").unwrap();
+        assert!(inbox.ephemeral, "auto-created from the nested label");
+        assert_eq!(
+            inbox.role,
+            Some("inbox"),
+            "an auto-created Inbox is still the inbox"
+        );
+    }
+
+    #[test]
+    fn role_closure_applies_to_explicit_top_level() {
+        let r = assemble(&["Inbox".to_owned()], inbox_role);
+        assert_eq!(r[0].role, Some("inbox"));
+    }
+
+    #[test]
+    fn order_is_parents_before_children() {
+        let r = assemble(
+            &[
+                "Zeta/Alpha".to_owned(),
+                "Alpha".to_owned(),
+                "Beta/Gamma/Delta".to_owned(),
+            ],
+            no_roles,
+        );
+        let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect();
+        assert_eq!(
+            names,
+            vec![
+                "Alpha",
+                "Beta",
+                "Beta/Gamma",
+                "Beta/Gamma/Delta",
+                "Zeta",
+                "Zeta/Alpha",
+            ]
+        );
+    }
+
+    #[test]
+    fn duplicate_inputs_dedup() {
+        let r = assemble(
+            &["Inbox".to_owned(), "Inbox".to_owned(), "Inbox".to_owned()],
+            no_roles,
+        );
+        assert_eq!(r.len(), 1);
+    }
+
+    #[test]
+    fn vanished_depth_sort_orders_leaves_first() {
+        let mut v = vec![
+            "Foo".to_owned(),
+            "Foo/Bar/Baz".to_owned(),
+            "Other".to_owned(),
+            "Foo/Bar".to_owned(),
+        ];
+        vanished_depth_sort(&mut v);
+        assert_eq!(v, vec!["Foo/Bar/Baz", "Foo/Bar", "Foo", "Other",]);
+    }
+
+    #[test]
+    fn vanished_depth_sort_handles_ties_lexicographically() {
+        let mut v = vec!["B".to_owned(), "A".to_owned(), "C".to_owned()];
+        vanished_depth_sort(&mut v);
+        assert_eq!(v, vec!["A", "B", "C"]);
+    }
+
+    #[test]
+    fn empty_input_yields_empty_output() {
+        let r = assemble(&[], no_roles);
+        assert!(r.is_empty());
+    }
+}

src/sync/import_takeout/walk.rs

@@ -0,0 +1,267 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashSet;
+use std::fs;
+use std::io;
+use std::os::unix::fs::MetadataExt;
+use std::path::{Path, PathBuf};
+
+use crate::logging::Logger;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum FileKind {
+    Mbox,
+    Ics,
+    Vcf,
+}
+
+impl FileKind {
+    fn from_extension(ext: &str) -> Option<FileKind> {
+        let lower = ext.to_ascii_lowercase();
+        match lower.as_str() {
+            "mbox" => Some(FileKind::Mbox),
+            "ics" => Some(FileKind::Ics),
+            "vcf" => Some(FileKind::Vcf),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct DiscoveredFile {
+    pub path: PathBuf,
+    pub kind: FileKind,
+}
+
+#[derive(Debug, Default)]
+pub struct WalkResult {
+    pub files: Vec<DiscoveredFile>,
+    pub symlink_cycles: u64,
+    pub io_failures: u64,
+}
+
+impl WalkResult {
+    pub fn by_kind(&self, kind: FileKind) -> impl Iterator<Item = &DiscoveredFile> {
+        self.files.iter().filter(move |f| f.kind == kind)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.files.is_empty()
+    }
+}
+
+pub fn walk(root: &Path) -> io::Result<WalkResult> {
+    walk_with_logger(root, Logger::from_flags(true, 0))
+}
+
+pub fn walk_with_logger(root: &Path, logger: Logger) -> io::Result<WalkResult> {
+    let mut result = WalkResult::default();
+    let mut visited: HashSet<(u64, u64)> = HashSet::new();
+    walk_dir(root, &mut result, &mut visited, logger)?;
+    result.files.sort_by(|a, b| a.path.cmp(&b.path));
+    Ok(result)
+}
+
+fn walk_dir(
+    dir: &Path,
+    result: &mut WalkResult,
+    visited: &mut HashSet<(u64, u64)>,
+    logger: Logger,
+) -> io::Result<()> {
+    let metadata = fs::metadata(dir)?;
+    let key = (metadata.dev(), metadata.ino());
+    if !visited.insert(key) {
+        result.symlink_cycles += 1;
+        logger.warn(&format!(
+            "takeout walk: symlink cycle broken at {dir:?} (target already visited)"
+        ));
+        return Ok(());
+    }
+
+    let entries = match fs::read_dir(dir) {
+        Ok(it) => it,
+        Err(e) => {
+            result.io_failures += 1;
+            logger.warn(&format!("takeout walk: read_dir {dir:?}: {e}"));
+            return Ok(());
+        }
+    };
+
+    for entry in entries {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(e) => {
+                result.io_failures += 1;
+                logger.warn(&format!("takeout walk: entry in {dir:?}: {e}"));
+                continue;
+            }
+        };
+        let path = entry.path();
+        let file_type = match entry.file_type() {
+            Ok(t) => t,
+            Err(e) => {
+                result.io_failures += 1;
+                logger.warn(&format!("takeout walk: file_type {path:?}: {e}"));
+                continue;
+            }
+        };
+        if file_type.is_dir() {
+            walk_dir(&path, result, visited, logger)?;
+            continue;
+        }
+        if file_type.is_symlink() {
+            let target_meta = match fs::metadata(&path) {
+                Ok(m) => m,
+                Err(e) => {
+                    result.io_failures += 1;
+                    logger.warn(&format!("takeout walk: symlink target {path:?}: {e}"));
+                    continue;
+                }
+            };
+            if target_meta.is_dir() {
+                walk_dir(&path, result, visited, logger)?;
+                continue;
+            }
+            if !target_meta.is_file() {
+                continue;
+            }
+        } else if !file_type.is_file() {
+            continue;
+        }
+        if let Some(kind) = classify(&path) {
+            result.files.push(DiscoveredFile { path, kind });
+        }
+    }
+    Ok(())
+}
+
+fn classify(path: &Path) -> Option<FileKind> {
+    let ext = path.extension()?.to_str()?;
+    FileKind::from_extension(ext)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use std::os::unix::fs::symlink;
+    use tempfile::TempDir;
+
+    fn touch(path: &Path) {
+        if let Some(parent) = path.parent() {
+            fs::create_dir_all(parent).unwrap();
+        }
+        fs::write(path, b"").unwrap();
+    }
+
+    #[test]
+    fn collects_only_matching_extensions() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("Mail/All.mbox"));
+        touch(&root.join("Calendar/cal.ics"));
+        touch(&root.join("Contacts/c.vcf"));
+        touch(&root.join("Calendar/meet_settings.json"));
+        touch(&root.join("archive_browser.html"));
+        touch(&root.join("Contacts/photo.jpg"));
+        let r = walk(root).unwrap();
+        assert_eq!(r.files.len(), 3);
+        let mut kinds: Vec<FileKind> = r.files.iter().map(|f| f.kind).collect();
+        kinds.sort_by_key(|k| format!("{k:?}"));
+        assert_eq!(kinds, vec![FileKind::Ics, FileKind::Mbox, FileKind::Vcf]);
+    }
+
+    #[test]
+    fn extension_match_is_case_insensitive() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("Inbox.MBOX"));
+        touch(&root.join("Cal.Ics"));
+        touch(&root.join("All.Vcf"));
+        let r = walk(root).unwrap();
+        assert_eq!(r.files.len(), 3);
+    }
+
+    #[test]
+    fn recurses_into_nested_directories() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("Takeout/Mail/a.mbox"));
+        touch(&root.join("Takeout/Calendar/deep/b.ics"));
+        touch(&root.join("Takeout/Contacts/All Contacts/c.vcf"));
+        let r = walk(root).unwrap();
+        assert_eq!(r.files.len(), 3);
+    }
+
+    #[test]
+    fn empty_or_non_matching_tree_returns_empty() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("readme.txt"));
+        touch(&root.join("nested/photo.jpg"));
+        let r = walk(root).unwrap();
+        assert!(r.is_empty());
+    }
+
+    #[test]
+    fn results_are_sorted_for_determinism() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("z.mbox"));
+        touch(&root.join("a.mbox"));
+        touch(&root.join("m.ics"));
+        let r = walk(root).unwrap();
+        let names: Vec<&str> = r
+            .files
+            .iter()
+            .map(|f| f.path.file_name().unwrap().to_str().unwrap())
+            .collect();
+        assert_eq!(names, vec!["a.mbox", "m.ics", "z.mbox"]);
+    }
+
+    #[test]
+    fn by_kind_filters_returned_iterator() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("a.mbox"));
+        touch(&root.join("b.mbox"));
+        touch(&root.join("c.ics"));
+        let r = walk(root).unwrap();
+        assert_eq!(r.by_kind(FileKind::Mbox).count(), 2);
+        assert_eq!(r.by_kind(FileKind::Ics).count(), 1);
+        assert_eq!(r.by_kind(FileKind::Vcf).count(), 0);
+    }
+
+    #[test]
+    fn symlink_to_directory_is_followed_once_not_infinitely() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join("real/a.mbox"));
+        symlink(root.join("real"), root.join("link-to-real")).unwrap();
+        let r = walk(root).unwrap();
+        assert_eq!(r.files.len(), 1);
+    }
+
+    #[test]
+    fn symlink_cycle_is_broken() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        fs::create_dir_all(root.join("a/b")).unwrap();
+        symlink(root.join("a"), root.join("a/b/loop")).unwrap();
+        let r = walk(root).unwrap();
+        assert!(r.symlink_cycles >= 1, "cycle counter incremented");
+    }
+
+    #[test]
+    fn dotted_directory_basenames_traverse_fine() {
+        let td = TempDir::new().unwrap();
+        let root = td.path();
+        touch(&root.join(".hidden/m.mbox"));
+        let r = walk(root).unwrap();
+        assert_eq!(r.files.len(), 1);
+    }
+}

src/sync/keys.rs

@@ -0,0 +1,269 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use serde_json::{Value, json};
+
+pub fn normalize_message_id(raw: &str) -> String {
+    let t = raw.trim();
+    let t = t.strip_prefix('<').unwrap_or(t);
+    let t = t.strip_suffix('>').unwrap_or(t);
+    t.trim().to_lowercase()
+}
+
+pub fn fold_name(name: &str) -> String {
+    name.trim().to_lowercase()
+}
+
+pub fn blake3_fields(fields: &[&str]) -> [u8; 32] {
+    let mut hasher = blake3::Hasher::new();
+    for (i, f) in fields.iter().enumerate() {
+        if i > 0 {
+            hasher.update(b"\x1f");
+        }
+        hasher.update(f.as_bytes());
+    }
+    *hasher.finalize().as_bytes()
+}
+
+pub fn blake3_bytes(bytes: &[u8]) -> [u8; 32] {
+    *blake3::hash(bytes).as_bytes()
+}
+
+pub fn identity_key(name: &str, email: &str) -> [u8; 32] {
+    blake3_fields(&[name, &email.to_lowercase()])
+}
+
+pub fn participant_identity_key(calendar_address: &str, name: &str) -> [u8; 32] {
+    blake3_fields(&[&calendar_address.to_lowercase(), name])
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum EmailKey {
+    MessageId(String),
+    Fallback([u8; 32]),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct EmailIndex {
+
+    pub mids: Vec<String>,
+
+    pub fb: [u8; 32],
+}
+
+fn join_addrs(addrs: &[String]) -> String {
+    addrs
+        .iter()
+        .map(|a| a.to_lowercase())
+        .collect::<Vec<_>>()
+        .join(",")
+}
+
+pub fn email_index(
+    message_ids: &[String],
+    from: &[String],
+    subject: &str,
+    sent_at: &str,
+    to: &[String],
+) -> EmailIndex {
+    let mut mids: Vec<String> = message_ids
+        .iter()
+        .map(|m| normalize_message_id(m))
+        .filter(|s| !s.is_empty())
+        .collect();
+    mids.sort();
+    mids.dedup();
+    let fb = blake3_fields(&[&join_addrs(from), subject, sent_at, &join_addrs(to)]);
+    EmailIndex { mids, fb }
+}
+
+pub fn index_to_json(idx: &EmailIndex) -> String {
+    json!({ "m": idx.mids, "f": hex(&idx.fb) }).to_string()
+}
+
+pub fn index_from_json(text: &str) -> EmailIndex {
+    let v: Value = serde_json::from_str(text).unwrap_or(Value::Null);
+    let mids = v
+        .get("m")
+        .and_then(Value::as_array)
+        .map(|a| {
+            a.iter()
+                .filter_map(|x| x.as_str().map(str::to_owned))
+                .collect()
+        })
+        .unwrap_or_default();
+    let fb = v
+        .get("f")
+        .and_then(Value::as_str)
+        .and_then(unhex)
+        .unwrap_or([0u8; 32]);
+    EmailIndex { mids, fb }
+}
+
+fn hex(bytes: &[u8; 32]) -> String {
+    let mut s = String::with_capacity(64);
+    for b in bytes {
+        use std::fmt::Write;
+        let _ = write!(s, "{b:02x}");
+    }
+    s
+}
+
+fn unhex(s: &str) -> Option<[u8; 32]> {
+    if s.len() != 64 {
+        return None;
+    }
+    let mut out = [0u8; 32];
+    for (i, slot) in out.iter_mut().enumerate() {
+        *slot = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).ok()?;
+    }
+    Some(out)
+}
+
+pub fn email_key(idx: &EmailIndex) -> EmailKey {
+    if idx.mids.is_empty() {
+        EmailKey::Fallback(idx.fb)
+    } else {
+        EmailKey::MessageId(idx.mids.join("\x1f"))
+    }
+}
+
+pub fn email_keys(indices: &[EmailIndex]) -> Vec<EmailKey> {
+    indices.iter().map(email_key).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn message_id_normalization_strips_brackets_and_lowercases() {
+        assert_eq!(normalize_message_id("  <[email protected]> "), "[email protected]");
+        assert_eq!(normalize_message_id("plain@x"), "plain@x");
+    }
+
+    #[test]
+    fn name_fold_is_case_insensitive() {
+        assert_eq!(fold_name(" Sales "), fold_name("sales"));
+        assert_ne!(fold_name("Sales"), fold_name("sale"));
+    }
+
+    #[test]
+    fn identity_collapse_ignores_signature() {
+        assert_eq!(
+            identity_key("Alice", "[email protected]"),
+            identity_key("Alice", "[email protected]")
+        );
+    }
+
+    #[test]
+    fn participant_identity_key_lowercases_address_only() {
+        let a = participant_identity_key("MailTo:User@Host", "Bob");
+        let b = participant_identity_key("mailto:user@host", "Bob");
+        assert_eq!(a, b);
+        assert_ne!(a, participant_identity_key("mailto:user@host", "bob"));
+    }
+
+    fn idx(mids: &[&str], from: &[&str], subj: &str, sent: &str, to: &[&str]) -> EmailIndex {
+        let m: Vec<String> = mids.iter().map(|s| (*s).to_owned()).collect();
+        let f: Vec<String> = from.iter().map(|s| (*s).to_owned()).collect();
+        let t: Vec<String> = to.iter().map(|s| (*s).to_owned()).collect();
+        email_index(&m, &f, subj, sent, &t)
+    }
+
+    #[test]
+    fn message_id_array_is_normalized_sorted_and_order_independent() {
+        let a = idx(&["<B@h>", "<a@H>"], &[], "", "", &[]);
+        let b = idx(&["<a@h>", "<b@h>"], &[], "", "", &[]);
+        assert_eq!(a.mids, vec!["a@h".to_owned(), "b@h".to_owned()]);
+        assert_eq!(a.mids, b.mids);
+    }
+
+    #[test]
+    fn unique_single_message_id_uses_message_id_key() {
+        let keys = email_keys(&[
+            idx(
+                &["<m1@h>"],
+                &["a@x"],
+                "Hi",
+                "2020-01-01T00:00:00Z",
+                &["b@y"],
+            ),
+            idx(
+                &["<m2@h>"],
+                &["a@x"],
+                "Yo",
+                "2020-01-02T00:00:00Z",
+                &["b@y"],
+            ),
+        ]);
+        assert_eq!(keys[0], EmailKey::MessageId("m1@h".to_owned()));
+        assert_eq!(keys[1], EmailKey::MessageId("m2@h".to_owned()));
+    }
+
+    #[test]
+    fn shared_message_id_matches_regardless_of_other_fields() {
+        let keys = email_keys(&[
+            idx(
+                &["<dup@h>"],
+                &["a@x"],
+                "S",
+                "2020-01-01T00:00:00Z",
+                &["b@y"],
+            ),
+            idx(
+                &["<dup@h>"],
+                &["c@x"],
+                "T",
+                "2020-01-02T00:00:00Z",
+                &["d@y"],
+            ),
+        ]);
+        assert_eq!(keys[0], EmailKey::MessageId("dup@h".to_owned()));
+        assert_eq!(keys[0], keys[1], "same Message-ID is the same message");
+    }
+
+    #[test]
+    fn only_absent_message_id_falls_back_multi_is_still_message_id() {
+        let keys = email_keys(&[
+            idx(&[], &["a@x"], "S", "2020-01-01T00:00:00Z", &["b@y"]),
+            idx(
+                &["<a@h>", "<b@h>"],
+                &["a@x"],
+                "S",
+                "2020-01-01T00:00:00Z",
+                &["b@y"],
+            ),
+        ]);
+        assert!(matches!(keys[0], EmailKey::Fallback(_)));
+        assert_eq!(keys[1], EmailKey::MessageId("a@h\u{1f}b@h".to_owned()));
+    }
+
+    #[test]
+    fn fallback_is_case_insensitive_on_addresses_and_offset_preserving_on_sent_at() {
+        let a = idx(&[], &["A@X"], "Subj", "2020-01-01T10:00:00+02:00", &["B@Y"]);
+        let b = idx(&[], &["a@x"], "Subj", "2020-01-01T10:00:00+02:00", &["b@y"]);
+        let c = idx(&[], &["a@x"], "Subj", "2020-01-01T08:00:00Z", &["b@y"]);
+        assert_eq!(a.fb, b.fb);
+        assert_ne!(a.fb, c.fb);
+    }
+
+    #[test]
+    fn index_json_roundtrips() {
+        let i = idx(&["<m@h>"], &["a@x"], "S", "2020-01-01T00:00:00Z", &["b@y"]);
+        let j = index_to_json(&i);
+        let back = index_from_json(&j);
+        assert_eq!(i, back);
+        assert_eq!(email_keys(&[i]), email_keys(&[back]));
+    }
+
+    #[test]
+    fn malformed_stored_json_degrades_to_empty_fallback() {
+        let back = index_from_json("{}");
+        assert!(back.mids.is_empty());
+        assert_eq!(back.fb, [0u8; 32]);
+    }
+}

src/sync/mod.rs

@@ -0,0 +1,120 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+pub mod emailmeta;
+pub mod export;
+pub mod import_dav;
+pub mod import_exchange_ews;
+pub mod import_exchange_graph;
+pub mod import_imap;
+pub mod import_jmap;
+pub mod import_maildir;
+pub mod import_managesieve;
+pub mod import_takeout;
+pub mod keys;
+pub mod prune;
+
+use std::path::PathBuf;
+
+use rusqlite::Connection;
+
+use crate::db;
+use crate::error::Error;
+
+pub(crate) fn table_name(ty: ObjectType) -> &'static str {
+    match ty {
+        ObjectType::Mailbox => "mailboxes",
+        ObjectType::Email => "emails",
+        ObjectType::Identity => "identities",
+        ObjectType::SieveScript => "sieve_scripts",
+        ObjectType::AddressBook => "address_books",
+        ObjectType::ContactCard => "contact_cards",
+        ObjectType::Calendar => "calendars",
+        ObjectType::CalendarEvent => "calendar_events",
+        ObjectType::ParticipantIdentity => "participant_identities",
+        ObjectType::FileNode => "file_nodes",
+    }
+}
+use crate::jmap::account::AccountSelector;
+use crate::jmap::http::{Auth, HttpClient, RetryPolicy};
+use crate::logging::Logger;
+use crate::types::ObjectType;
+
+pub struct CommonConfig {
+    pub archive: PathBuf,
+    pub threads: usize,
+    pub dry_run: bool,
+    pub max_retries: u32,
+    pub allow_invalid_certs: bool,
+    pub logger: Logger,
+}
+
+pub struct ConnectConfig {
+    pub url: String,
+    pub auth: Auth,
+    pub account: AccountSelector,
+}
+
+pub struct ImportConfig {
+    pub connect: ConnectConfig,
+    pub objects: Option<Vec<ObjectType>>,
+    pub allow_source_change: bool,
+}
+
+pub struct ExportConfig {
+    pub connect: ConnectConfig,
+    pub objects: Option<Vec<ObjectType>>,
+    pub prune: bool,
+    pub yes: bool,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct TypeCounts {
+    pub created: u64,
+    pub fetched: u64,
+    pub deleted: u64,
+    pub skipped: u64,
+    pub failed: u64,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct Summary {
+    pub per_type: Vec<(&'static str, TypeCounts)>,
+    pub retries_observed: u64,
+    pub retry_after_sleeps: u64,
+}
+
+impl Summary {
+    pub fn any_failed(&self) -> bool {
+        self.per_type.iter().any(|(_, c)| c.failed > 0)
+    }
+}
+
+pub struct Context {
+    pub conn: Connection,
+    pub client: HttpClient,
+    pub common: CommonConfig,
+}
+
+impl Context {
+    pub fn open(common: CommonConfig, connect: &ConnectConfig) -> Result<Context, Error> {
+        let conn = db::init::open(&common.archive)?;
+        let client = HttpClient::new(
+            connect.auth.clone(),
+            RetryPolicy::new(common.max_retries),
+            common.allow_invalid_certs,
+        );
+        Ok(Context {
+            conn,
+            client,
+            common,
+        })
+    }
+
+    pub fn dry_run(&self) -> bool {
+        self.common.dry_run
+    }
+}

src/sync/prune.rs

@@ -0,0 +1,124 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::{HashMap, HashSet};
+
+#[derive(Debug, Clone)]
+pub struct TargetObj {
+    pub id: String,
+    pub matched: bool,
+    pub protected: bool,
+    pub may_delete: bool,
+    pub parent: Option<String>,
+}
+
+fn protected_with_ancestors(objs: &[TargetObj]) -> HashSet<String> {
+    let by_id: HashMap<&str, &TargetObj> = objs.iter().map(|o| (o.id.as_str(), o)).collect();
+    let mut protected: HashSet<String> = HashSet::new();
+    for o in objs {
+        if o.protected {
+            let mut cur = Some(o.id.clone());
+            while let Some(id) = cur {
+                if !protected.insert(id.clone()) {
+                    break;
+                }
+                cur = by_id
+                    .get(id.as_str())
+                    .and_then(|x| x.parent.clone())
+                    .filter(|p| by_id.contains_key(p.as_str()));
+            }
+        }
+    }
+    protected
+}
+
+fn depth(id: &str, by_id: &HashMap<&str, &TargetObj>) -> usize {
+    let mut d = 0;
+    let mut cur = by_id.get(id).and_then(|o| o.parent.clone());
+    let mut seen: HashSet<String> = HashSet::new();
+    while let Some(p) = cur {
+        if !seen.insert(p.clone()) || !by_id.contains_key(p.as_str()) {
+            break;
+        }
+        d += 1;
+        cur = by_id.get(p.as_str()).and_then(|o| o.parent.clone());
+    }
+    d
+}
+
+pub fn candidates(objs: &[TargetObj], tree: bool) -> Vec<String> {
+    let protected = protected_with_ancestors(objs);
+    let by_id: HashMap<&str, &TargetObj> = objs.iter().map(|o| (o.id.as_str(), o)).collect();
+    let mut out: Vec<&TargetObj> = objs
+        .iter()
+        .filter(|o| !o.matched && o.may_delete && !protected.contains(&o.id))
+        .collect();
+    if tree {
+        out.sort_by(|a, b| {
+            depth(&b.id, &by_id)
+                .cmp(&depth(&a.id, &by_id))
+                .then_with(|| a.id.cmp(&b.id))
+        });
+    } else {
+        out.sort_by(|a, b| a.id.cmp(&b.id));
+    }
+    out.into_iter().map(|o| o.id.clone()).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn o(id: &str, matched: bool, protected: bool, parent: Option<&str>) -> TargetObj {
+        TargetObj {
+            id: id.to_owned(),
+            matched,
+            protected,
+            may_delete: true,
+            parent: parent.map(str::to_owned),
+        }
+    }
+
+    #[test]
+    fn unmatched_unprotected_are_candidates() {
+        let objs = vec![
+            o("a", true, false, None),
+            o("b", false, false, None),
+            o("c", false, true, None),
+        ];
+        assert_eq!(candidates(&objs, false), vec!["b".to_owned()]);
+    }
+
+    #[test]
+    fn protected_excludes_ancestors() {
+        let objs = vec![
+            o("root", false, false, None),
+            o("mid", false, false, Some("root")),
+            o("leaf", false, true, Some("mid")),
+        ];
+        assert!(candidates(&objs, true).is_empty());
+    }
+
+    #[test]
+    fn may_delete_false_is_skipped() {
+        let mut x = o("x", false, false, None);
+        x.may_delete = false;
+        assert!(candidates(&[x], false).is_empty());
+    }
+
+    #[test]
+    fn tree_destroy_is_leaf_first() {
+        let objs = vec![
+            o("root", false, false, None),
+            o("child", false, false, Some("root")),
+            o("grand", false, false, Some("child")),
+        ];
+        assert_eq!(
+            candidates(&objs, true),
+            vec!["grand".to_owned(), "child".to_owned(), "root".to_owned()]
+        );
+    }
+}