Initial commit

src/sync/import_exchange_ews/attachments.rs

@@ -0,0 +1,103 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD;
+
+use crate::db::blobs;
+use crate::error::Error;
+use crate::exchange_ews::error::EwsError;
+use crate::exchange_ews::parse::parse_get_attachment_inline;
+use crate::exchange_ews::xml::get_attachment_body;
+
+use super::items::ItemRunCtx;
+
+#[derive(Debug, Clone)]
+pub struct FetchedAttachment {
+    pub attachment_id: String,
+    pub bytes: Vec<u8>,
+    pub media_type: String,
+}
+
+pub fn fetch_contact_photo(
+    conn: &rusqlite::Connection,
+    ctx: &ItemRunCtx<'_>,
+    attachment_id: &str,
+) -> Result<Option<(i64, String)>, Error> {
+    let Some(att) = fetch_attachment_bytes(ctx, attachment_id).map_err(Error::from)? else {
+        return Ok(None);
+    };
+    let blob_id = intern_attachment(conn, &att.bytes)?;
+    Ok(Some((blob_id, att.media_type)))
+}
+
+pub fn fetch_attachment_bytes(
+    ctx: &ItemRunCtx<'_>,
+    attachment_id: &str,
+) -> Result<Option<FetchedAttachment>, EwsError> {
+    let mut out = fetch_attachments(ctx, &[attachment_id])?;
+    Ok(out.pop())
+}
+
+pub fn fetch_attachments(
+    ctx: &ItemRunCtx<'_>,
+    attachment_ids: &[&str],
+) -> Result<Vec<FetchedAttachment>, EwsError> {
+    if attachment_ids.is_empty() {
+        return Ok(Vec::new());
+    }
+    let batch = ctx.attachment_batch.max(1);
+    let mut out: Vec<FetchedAttachment> = Vec::with_capacity(attachment_ids.len());
+    for chunk in attachment_ids.chunks(batch) {
+        let body = get_attachment_body(chunk);
+        let resp = ctx.client.call(ctx.url, "GetAttachment", &body)?;
+        let inline = parse_get_attachment_inline(&resp.body)?;
+        for att in inline {
+            let cleaned = strip_ascii_whitespace(att.content_base64.as_bytes());
+            if cleaned.is_empty() {
+                continue;
+            }
+            let bytes = STANDARD.decode(&cleaned).map_err(|e| {
+                EwsError::Malformed(format!("attachment {}: base64: {e}", att.attachment_id))
+            })?;
+            let media_type = att
+                .content_type
+                .unwrap_or_else(|| "application/octet-stream".to_owned());
+            out.push(FetchedAttachment {
+                attachment_id: att.attachment_id,
+                bytes,
+                media_type,
+            });
+        }
+    }
+    Ok(out)
+}
+
+pub fn intern_attachment(conn: &rusqlite::Connection, bytes: &[u8]) -> Result<i64, Error> {
+    blobs::intern_blob(conn, bytes).map_err(|e| Error::Partial(e.to_string()))
+}
+
+fn strip_ascii_whitespace(input: &[u8]) -> Vec<u8> {
+    let mut out = Vec::with_capacity(input.len());
+    for b in input {
+        if !matches!(b, b' ' | b'\t' | b'\n' | b'\r') {
+            out.push(*b);
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::strip_ascii_whitespace;
+
+    #[test]
+    fn strips_whitespace_efficiently() {
+        let input = b"AB CD\nEF\tGH\r\nIJ";
+        let cleaned = strip_ascii_whitespace(input);
+        assert_eq!(cleaned, b"ABCDEFGHIJ");
+    }
+}

src/sync/import_exchange_ews/calendar.rs

@@ -0,0 +1,292 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::{Connection, params};
+use serde_json::Value;
+
+use crate::db::exchange_ews_ids;
+use crate::error::Error;
+use crate::exchange_ews::calendar_map::to_jscalendar;
+use crate::exchange_ews::parse::{CalendarItemRaw, parse_calendar_item};
+use crate::exchange_ews::types::{CalendarItemType, ItemId};
+use crate::exchange_ews::xml::ItemShape;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+
+use super::attachments::{fetch_attachments, intern_attachment};
+use super::folders::FolderPlan;
+use super::items::{
+    EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
+};
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    plan: &FolderPlan,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for folder in &plan.calendar {
+        let folder_id = &folder.folder.folder_id;
+        let local_folder_id = match exchange_ews_ids::local_for_item(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            &folder_id.id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        {
+            Some(id) => id,
+            None => continue,
+        };
+        if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) {
+            ctx.logger
+                .warn(&format!("calendar folder {} failed: {}", folder_id.id, e));
+            counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn reconcile_one(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    folder: &crate::exchange_ews::types::FolderId,
+    local_folder_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let prior = exchange_ews_ids::get_sync_state(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CALENDAR,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let mut outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?;
+    outcome.items.retain(|s| {
+        matches!(
+            s.element.to_ascii_lowercase().as_str(),
+            "calendaritem" | "item"
+        )
+    });
+    let local = exchange_ews_ids::items_in_folder(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CALENDAR_EVENT,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let plan = plan_for(&outcome, &local);
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS calendar folder {}: new={} changed={} vanished={}",
+            folder.id,
+            plan.new.len(),
+            plan.present_changed.len(),
+            plan.vanished.len()
+        );
+    }
+    let mut to_fetch: Vec<ItemId> = plan.new.clone();
+    for (id, _) in &plan.present_changed {
+        to_fetch.push(id.clone());
+    }
+    if !to_fetch.is_empty() {
+        let outcome = get_items(ctx, ItemShape::CalendarItem, &to_fetch).map_err(Error::from)?;
+        counts.failed += outcome.failed_items;
+        for msg in outcome.messages {
+            if !msg.success {
+                if matches!(
+                    msg.response_code,
+                    crate::exchange_ews::types::ResponseCode::ItemNotFound
+                ) {
+                    counts.skipped += 1;
+                } else {
+                    counts.failed += 1;
+                    ctx.logger
+                        .warn(&format!("GetItem (calendar) error: {}", msg.response_code));
+                }
+                continue;
+            }
+            let parsed = parse_calendar_item(&msg.inner_xml).map_err(Error::from)?;
+            if parsed.id.id.is_empty() {
+                counts.failed += 1;
+                continue;
+            }
+            if matches!(
+                parsed.calendar_item_type,
+                Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception)
+            ) {
+                counts.skipped += 1;
+                continue;
+            }
+            let existing = plan
+                .present_changed
+                .iter()
+                .find(|(id, _)| id.id == parsed.id.id)
+                .map(|(_, local)| *local);
+            apply_event(
+                conn,
+                ctx,
+                &parsed,
+                local_folder_id,
+                &folder.id,
+                existing,
+                counts,
+            )?;
+        }
+    }
+    delete_vanished(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CALENDAR_EVENT,
+        "calendar_events",
+        &plan.vanished,
+        counts,
+    )?;
+    if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode {
+        exchange_ews_ids::set_sync_state(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            &folder.id,
+            new_sync_state,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn apply_event(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    raw: &CalendarItemRaw,
+    local_folder_id: i64,
+    folder_id: &str,
+    existing_local_id: Option<i64>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let mut built = to_jscalendar(raw);
+    attach_calendar_links(conn, ctx, raw, &mut built.data)?;
+    let calendar_ids = serde_json::json!([local_folder_id]).to_string();
+    let data = built.data.to_string();
+    let is_draft = if built.is_draft { 1 } else { 0 };
+    let use_default_alerts = if built.use_default_alerts { 1 } else { 0 };
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    match existing_local_id {
+        Some(id) => {
+            tx.execute(
+                "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, \
+                 use_default_alerts = ?3, data = ?4, data_type = 'Event' WHERE id = ?5",
+                params![calendar_ids, is_draft, use_default_alerts, data, id],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            exchange_ews_ids::update_change_key(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CALENDAR_EVENT,
+                &raw.id.id,
+                &raw.id.change_key,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.fetched += 1;
+        }
+        None => {
+            tx.execute(
+                "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) \
+                 VALUES (?1, ?2, ?3, ?4, 'Event')",
+                params![calendar_ids, is_draft, use_default_alerts, data],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            let new_id = tx.last_insert_rowid();
+            exchange_ews_ids::insert(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CALENDAR_EVENT,
+                folder_id,
+                &raw.id.id,
+                &raw.id.change_key,
+                new_id,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.created += 1;
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}
+
+fn attach_calendar_links(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    raw: &CalendarItemRaw,
+    data: &mut Value,
+) -> Result<(), Error> {
+    let refs: Vec<_> = raw
+        .attachments
+        .iter()
+        .filter(|a| !a.is_item_attachment && !a.attachment_id.is_empty())
+        .collect();
+    if refs.is_empty() {
+        return Ok(());
+    }
+    let ids: Vec<&str> = refs.iter().map(|a| a.attachment_id.as_str()).collect();
+    let fetched = match fetch_attachments(ctx, &ids) {
+        Ok(f) => f,
+        Err(e) => {
+            ctx.logger.warn(&format!(
+                "calendar attachments fetch failed: {e}; continuing without enclosures"
+            ));
+            return Ok(());
+        }
+    };
+    let mut links = serde_json::Map::new();
+    for (idx, att) in (1u32..).zip(fetched) {
+        let blob_id = intern_attachment(conn, &att.bytes)?;
+        let declared = refs
+            .iter()
+            .find(|r| r.attachment_id == att.attachment_id)
+            .and_then(|r| r.content_type.as_deref());
+        let name = refs
+            .iter()
+            .find(|r| r.attachment_id == att.attachment_id)
+            .and_then(|r| r.name.clone());
+        let key = idx.to_string();
+        let mut entry = serde_json::Map::new();
+        entry.insert("@type".to_owned(), Value::String("Link".to_owned()));
+        entry.insert("@blob".to_owned(), Value::from(blob_id));
+        entry.insert(
+            "contentType".to_owned(),
+            Value::String(content_type_or_from(declared, &att.media_type)),
+        );
+        if let Some(n) = name {
+            entry.insert("title".to_owned(), Value::String(n));
+        }
+        entry.insert("rel".to_owned(), Value::String("enclosure".to_owned()));
+        links.insert(key, Value::Object(entry));
+    }
+    if links.is_empty() {
+        return Ok(());
+    }
+    if let Some(obj) = data.as_object_mut() {
+        if let Some(existing) = obj.get_mut("links").and_then(Value::as_object_mut) {
+            for (k, v) in links {
+                existing.insert(k, v);
+            }
+        } else {
+            obj.insert("links".to_owned(), Value::Object(links));
+        }
+    }
+    Ok(())
+}
+
+fn content_type_or_from(declared: Option<&str>, fallback: &str) -> String {
+    declared
+        .filter(|s| !s.is_empty())
+        .map(str::to_owned)
+        .unwrap_or_else(|| fallback.to_owned())
+}

src/sync/import_exchange_ews/contacts.rs

@@ -0,0 +1,227 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::{Connection, params};
+
+use crate::db::exchange_ews_ids;
+use crate::error::Error;
+use crate::exchange_ews::contact_map::{synthetic_uid, to_jscontact};
+use crate::exchange_ews::parse::{ContactItemRaw, parse_contact_item};
+use crate::exchange_ews::types::ItemId;
+use crate::exchange_ews::xml::ItemShape;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+
+use super::attachments::fetch_contact_photo;
+use super::folders::FolderPlan;
+use super::items::{
+    EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
+};
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    plan: &FolderPlan,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for folder in &plan.contacts {
+        let folder_id = &folder.folder.folder_id;
+        let local_folder_id = match exchange_ews_ids::local_for_item(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            &folder_id.id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        {
+            Some(id) => id,
+            None => continue,
+        };
+        if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) {
+            ctx.logger
+                .warn(&format!("contact folder {} failed: {}", folder_id.id, e));
+            counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn reconcile_one(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    folder: &crate::exchange_ews::types::FolderId,
+    local_folder_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let prior = exchange_ews_ids::get_sync_state(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::ADDRESS_BOOK,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?;
+    let local = exchange_ews_ids::items_in_folder(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CONTACT_CARD,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let plan = plan_for(&outcome, &local);
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS contacts folder {}: new={} changed={} vanished={}",
+            folder.id,
+            plan.new.len(),
+            plan.present_changed.len(),
+            plan.vanished.len()
+        );
+    }
+    let mut to_fetch: Vec<ItemId> = plan.new.clone();
+    for (id, _) in &plan.present_changed {
+        to_fetch.push(id.clone());
+    }
+    if !to_fetch.is_empty() {
+        let outcome = get_items(ctx, ItemShape::Contact, &to_fetch).map_err(Error::from)?;
+        counts.failed += outcome.failed_items;
+        for msg in outcome.messages {
+            if !msg.success {
+                if matches!(
+                    msg.response_code,
+                    crate::exchange_ews::types::ResponseCode::ItemNotFound
+                ) {
+                    counts.skipped += 1;
+                } else {
+                    counts.failed += 1;
+                    ctx.logger
+                        .warn(&format!("GetItem (contact) error: {}", msg.response_code));
+                }
+                continue;
+            }
+            let parsed = parse_contact_item(&msg.inner_xml).map_err(Error::from)?;
+            if parsed.id.id.is_empty() {
+                counts.failed += 1;
+                continue;
+            }
+            let existing = plan
+                .present_changed
+                .iter()
+                .find(|(id, _)| id.id == parsed.id.id)
+                .map(|(_, local)| *local);
+            apply_contact(
+                conn,
+                ctx,
+                &parsed,
+                local_folder_id,
+                &folder.id,
+                existing,
+                counts,
+            )?;
+        }
+    }
+    delete_vanished(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::CONTACT_CARD,
+        "contact_cards",
+        &plan.vanished,
+        counts,
+    )?;
+    if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode {
+        exchange_ews_ids::set_sync_state(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            &folder.id,
+            new_sync_state,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn apply_contact(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    item: &ContactItemRaw,
+    local_folder_id: i64,
+    folder_id: &str,
+    existing_local_id: Option<i64>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let mut card = to_jscontact(item);
+    if let Some(photo_att) = item.attachments.iter().find(|a| a.is_contact_photo) {
+        match fetch_contact_photo(conn, ctx, &photo_att.attachment_id) {
+            Ok(Some((blob_id, media_type))) => {
+                let mut media = serde_json::Map::new();
+                media.insert(
+                    "photo".to_owned(),
+                    serde_json::json!({
+                        "@type": "Media",
+                        "kind": "photo",
+                        "@blob": blob_id,
+                        "mediaType": media_type,
+                    }),
+                );
+                if let Some(m) = card.as_object_mut() {
+                    m.insert("media".to_owned(), serde_json::Value::Object(media));
+                }
+            }
+            Ok(None) => {}
+            Err(e) => {
+                ctx.logger
+                    .warn(&format!("contact photo fetch failed: {e}; continuing"));
+            }
+        }
+    }
+    let uid = synthetic_uid(&item.id.id);
+    let address_book_ids = serde_json::json!([local_folder_id]).to_string();
+    let data = card.to_string();
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    match existing_local_id {
+        Some(id) => {
+            tx.execute(
+                "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 WHERE id = ?4",
+                params![uid, address_book_ids, data, id],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            exchange_ews_ids::update_change_key(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CONTACT_CARD,
+                &item.id.id,
+                &item.id.change_key,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.fetched += 1;
+        }
+        None => {
+            tx.execute(
+                "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)",
+                params![uid, address_book_ids, data],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            let new_id = tx.last_insert_rowid();
+            exchange_ews_ids::insert(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::CONTACT_CARD,
+                folder_id,
+                &item.id.id,
+                &item.id.change_key,
+                new_id,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.created += 1;
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}

src/sync/import_exchange_ews/coordinator.rs

@@ -0,0 +1,521 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use rusqlite::Connection;
+
+use crate::db;
+use crate::db::sources::SourceKey;
+use crate::error::Error;
+use crate::exchange_ews::EwsClient;
+use crate::exchange_ews::autodiscover::{DiscoveryResult, discover};
+use crate::exchange_ews::oauth::{AcquiredToken, OAuthFlow, acquire};
+use crate::exchange_ews::types::MailboxKind;
+use crate::jmap::http::{Auth, RetryPolicy};
+use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS};
+use crate::sync::{CommonConfig, Summary, TypeCounts};
+
+use super::folders::{self, plan_folders};
+use super::{calendar, contacts, messages};
+
+#[derive(Debug, Clone)]
+pub enum EwsAuth {
+    Basic { user: String, password: String },
+    Bearer { token: String },
+    OAuth(OAuthFlow),
+}
+
+#[derive(Debug, Clone)]
+pub struct EwsImportConfig {
+    pub url: Option<String>,
+    pub mailbox: Option<String>,
+    pub mailbox_kind: MailboxKind,
+    pub auth: EwsAuth,
+    pub ews_connections: usize,
+    pub getitem_batch: usize,
+    pub attachment_batch: usize,
+    pub use_syncfolderitems: bool,
+    pub allow_source_change: bool,
+}
+
+pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Error> {
+    let logger = common.logger;
+    let mut conn = db::init::open(&common.archive)?;
+
+    let (auth, acquired) = resolve_auth(&config.auth, common.allow_invalid_certs)?;
+    let discovery = run_autodiscover(&config, &acquired, common.allow_invalid_certs)?;
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS discovery: url={} source={:?}",
+            discovery.ews_url, discovery.source
+        );
+    }
+    let mailbox = resolve_mailbox(&config, &acquired)?;
+    let account_id = synthetic_account_id(&mailbox, config.mailbox_kind);
+    let session_url = normalise_ews_url(&discovery.ews_url);
+    enforce_basic_auth_policy(&config.auth, &session_url)?;
+    let key = SourceKey {
+        kind: "exchange_ews".to_owned(),
+        session_url: session_url.clone(),
+        account_id: account_id.clone(),
+    };
+
+    if !common.dry_run
+        && let Some((url, acc)) =
+            db::sources::conflicting_source(&conn, "exchange_ews", &session_url, &account_id)?
+        && !config.allow_source_change
+    {
+        return Err(Error::SourceChange(format!(
+            "archive already records exchange_ews source ({url}, account {acc}); \
+             pass --allow-source-change to import a different account"
+        )));
+    }
+
+    let client = EwsClient::new(
+        auth,
+        RetryPolicy::new(common.max_retries),
+        common.allow_invalid_certs,
+    );
+    client.set_logger(logger);
+    if matches!(config.mailbox_kind, MailboxKind::PublicFolders) {
+        client.set_anchor_mailbox(None);
+    } else {
+        client.set_anchor_mailbox(Some(mailbox.clone()));
+    }
+    if let EwsAuth::OAuth(OAuthFlow::ClientCredentials { .. }) = &config.auth {
+        client.set_impersonation(Some(mailbox.clone()));
+    }
+    spawn_token_refresher(
+        &client,
+        &config.auth,
+        &acquired,
+        common.allow_invalid_certs,
+        logger,
+    );
+
+    let username = match &config.auth {
+        EwsAuth::Basic { user, .. } => user.clone(),
+        EwsAuth::Bearer { .. } | EwsAuth::OAuth(_) => acquired
+            .as_ref()
+            .and_then(|a| a.upn.clone())
+            .unwrap_or_else(|| mailbox.clone()),
+    };
+
+    if common.dry_run {
+        let summary = run_dry(&conn, &client, &session_url, &config, logger)?;
+        return Ok(summary);
+    }
+
+    let account_name = acquired.as_ref().and_then(|a| a.name.clone());
+    let source_id = db::sources::upsert_source(&conn, &key, account_name.as_deref(), &username)?;
+
+    let mut summary = Summary::default();
+    let mut mailbox_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut addressbook_counts = TypeCounts::default();
+    let mut email_counts = TypeCounts::default();
+    let mut calendar_event_counts = TypeCounts::default();
+    let mut contact_counts = TypeCounts::default();
+
+    let plan =
+        plan_folders(&client, &session_url, config.mailbox_kind, logger).map_err(Error::from)?;
+
+    folders::reconcile(
+        &mut conn,
+        source_id,
+        &plan,
+        &mut mailbox_counts,
+        &mut calendar_counts,
+        &mut addressbook_counts,
+        logger,
+    )?;
+
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "import: mailbox={} calendar={} addressbook={}",
+            mailbox_counts.created + mailbox_counts.fetched,
+            calendar_counts.created + calendar_counts.fetched,
+            addressbook_counts.created + addressbook_counts.fetched,
+        );
+    }
+
+    let item_ctx = super::items::ItemRunCtx {
+        client: &client,
+        url: &session_url,
+        source_id,
+        batch_size: config.getitem_batch.max(1),
+        attachment_batch: config.attachment_batch.max(1),
+        connections: config.ews_connections.clamp(1, 8),
+        use_syncfolderitems: config.use_syncfolderitems,
+        logger,
+    };
+
+    messages::reconcile_all(&mut conn, &item_ctx, &plan, &mut email_counts)?;
+    contacts::reconcile_all(&mut conn, &item_ctx, &plan, &mut contact_counts)?;
+    calendar::reconcile_all(&mut conn, &item_ctx, &plan, &mut calendar_event_counts)?;
+
+    summary.per_type.push(("mailbox", mailbox_counts));
+    summary.per_type.push(("calendar", calendar_counts));
+    summary.per_type.push(("addressbook", addressbook_counts));
+    summary.per_type.push(("email", email_counts));
+    summary
+        .per_type
+        .push(("calendarevent", calendar_event_counts));
+    summary.per_type.push(("contactcard", contact_counts));
+
+    if !summary.any_failed()
+        && let Err(e) = run_gc(&conn)
+    {
+        logger.warn(&format!("blob GC skipped: {e}"));
+    }
+
+    summary.retries_observed = client.retries_observed();
+    summary.retry_after_sleeps = client.retry_after_sleeps();
+    Ok(summary)
+}
+
+fn run_dry(
+    _conn: &Connection,
+    client: &EwsClient,
+    session_url: &str,
+    config: &EwsImportConfig,
+    logger: crate::logging::Logger,
+) -> Result<Summary, Error> {
+    let plan =
+        plan_folders(client, session_url, config.mailbox_kind, logger).map_err(Error::from)?;
+    let mut summary = Summary::default();
+    let mut mailbox_counts = TypeCounts::default();
+    let mut calendar_counts = TypeCounts::default();
+    let mut addressbook_counts = TypeCounts::default();
+    mailbox_counts.created = plan.mail.len() as u64;
+    calendar_counts.created = plan.calendar.len() as u64;
+    addressbook_counts.created = plan.contacts.len() as u64;
+    if logger.enabled(LEVEL_DEFAULT) {
+        eprintln!(
+            "dry-run: mailbox={} calendar={} addressbook={}",
+            mailbox_counts.created, calendar_counts.created, addressbook_counts.created
+        );
+    }
+    summary.per_type.push(("mailbox", mailbox_counts));
+    summary.per_type.push(("calendar", calendar_counts));
+    summary.per_type.push(("addressbook", addressbook_counts));
+    Ok(summary)
+}
+
+fn resolve_auth(
+    auth: &EwsAuth,
+    allow_invalid_certs: bool,
+) -> Result<(Auth, Option<AcquiredToken>), Error> {
+    match auth {
+        EwsAuth::Basic { user, password } => Ok((
+            Auth::Basic {
+                user: user.clone(),
+                password: password.clone(),
+            },
+            None,
+        )),
+        EwsAuth::Bearer { token } => {
+            let acq = acquire(
+                &OAuthFlow::PreAcquired {
+                    token: token.clone(),
+                },
+                allow_invalid_certs,
+            )
+            .map_err(Error::from)?;
+            Ok((
+                Auth::Bearer {
+                    token: acq.access_token.clone(),
+                },
+                Some(acq),
+            ))
+        }
+        EwsAuth::OAuth(flow) => {
+            let acq = acquire(flow, allow_invalid_certs).map_err(Error::from)?;
+            Ok((
+                Auth::Bearer {
+                    token: acq.access_token.clone(),
+                },
+                Some(acq),
+            ))
+        }
+    }
+}
+
+fn run_autodiscover(
+    config: &EwsImportConfig,
+    acquired: &Option<AcquiredToken>,
+    allow_invalid_certs: bool,
+) -> Result<DiscoveryResult, Error> {
+    let email = config
+        .mailbox
+        .clone()
+        .or_else(|| acquired.as_ref().and_then(|a| a.upn.clone()));
+    let result = discover(
+        config.url.as_deref(),
+        email.as_deref(),
+        None,
+        allow_invalid_certs,
+    )
+    .map_err(Error::from)?;
+    Ok(result)
+}
+
+fn resolve_mailbox(
+    config: &EwsImportConfig,
+    acquired: &Option<AcquiredToken>,
+) -> Result<String, Error> {
+    if let Some(mb) = config.mailbox.as_ref() {
+        return Ok(mb.clone());
+    }
+    if let Some(acq) = acquired.as_ref()
+        && let Some(upn) = acq.upn.as_ref()
+    {
+        return Ok(upn.clone());
+    }
+    if let EwsAuth::Basic { user, .. } = &config.auth
+        && user.contains('@')
+    {
+        return Ok(user.clone());
+    }
+    Err(Error::Account(
+        "could not resolve mailbox SMTP; pass --mailbox".to_owned(),
+    ))
+}
+
+pub fn synthetic_account_id(mailbox: &str, kind: MailboxKind) -> String {
+    match kind {
+        MailboxKind::Primary => mailbox.to_owned(),
+        MailboxKind::Archive => format!("{mailbox}#archive"),
+        MailboxKind::PublicFolders => {
+            let domain = mailbox.split('@').nth(1).unwrap_or("");
+            format!("__public_folders__@{domain}")
+        }
+    }
+}
+
+pub fn normalise_ews_url(raw: &str) -> String {
+    let Ok(parsed) = url::Url::parse(raw) else {
+        return raw.to_owned();
+    };
+    let scheme = parsed.scheme().to_ascii_lowercase();
+    let host = parsed.host_str().unwrap_or("").to_ascii_lowercase();
+    let default_port = match scheme.as_str() {
+        "https" => 443,
+        "http" => 80,
+        _ => 0,
+    };
+    let port = parsed.port().filter(|p| *p != default_port);
+    let path = if parsed.path().is_empty() || parsed.path() == "/" {
+        "/EWS/Exchange.asmx".to_owned()
+    } else {
+        let last = parsed.path().rsplit('/').next().unwrap_or("");
+        if last.eq_ignore_ascii_case("Exchange.asmx") {
+            let mut p = String::new();
+            let mut parts: Vec<&str> = parsed.path().split('/').collect();
+            if let Some(last_part) = parts.last_mut() {
+                *last_part = "Exchange.asmx";
+            }
+            if let Some(second_last_idx) = parts.len().checked_sub(2)
+                && parts[second_last_idx].eq_ignore_ascii_case("EWS")
+            {
+                parts[second_last_idx] = "EWS";
+            }
+            p.push_str(&parts.join("/"));
+            p
+        } else {
+            parsed.path().to_owned()
+        }
+    };
+    match port {
+        Some(p) => format!("{scheme}://{host}:{p}{path}"),
+        None => format!("{scheme}://{host}{path}"),
+    }
+}
+
+fn enforce_basic_auth_policy(auth: &EwsAuth, url: &str) -> Result<(), Error> {
+    if let EwsAuth::Basic { .. } = auth {
+        let parsed = url::Url::parse(url)
+            .map_err(|e| Error::Usage(format!("invalid EWS URL {url:?}: {e}")))?;
+        if parsed.scheme() != "https" {
+            return Err(Error::Connection(
+                "EWS requires https:// (basic auth refused on cleartext)".to_owned(),
+            ));
+        }
+        let host = parsed.host_str().unwrap_or("").to_ascii_lowercase();
+        let blocked = [
+            "outlook.office365.com",
+            "outlook.office.com",
+            "outlook.office365.us",
+            "office365.us",
+        ];
+        if blocked.iter().any(|b| host == *b || host.ends_with(b)) {
+            return Err(Error::Connection(
+                "Basic auth is disabled on Exchange Online; use --auth-bearer".to_owned(),
+            ));
+        }
+    }
+    Ok(())
+}
+
+fn spawn_token_refresher(
+    client: &EwsClient,
+    auth: &EwsAuth,
+    initial: &Option<AcquiredToken>,
+    allow_invalid_certs: bool,
+    logger: crate::logging::Logger,
+) {
+    let flow = match auth {
+        EwsAuth::OAuth(flow) => flow.clone(),
+        _ => return,
+    };
+    let exp = initial.as_ref().and_then(|t| {
+        crate::exchange_ews::oauth::decode_jwt_claims(&t.access_token).and_then(|c| c.exp)
+    });
+    let Some(exp_secs) = exp else {
+        return;
+    };
+    let client = client.clone();
+    let initial_refresh = initial.as_ref().and_then(|t| t.refresh_token.clone());
+    std::thread::Builder::new()
+        .name("vandelay-ews-token-refresh".to_owned())
+        .spawn(move || {
+            let mut next_exp = exp_secs;
+            let mut refresh_token = initial_refresh;
+            loop {
+                let now = std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .map(|d| d.as_secs())
+                    .unwrap_or(0);
+                let refresh_at = next_exp.saturating_sub(5 * 60);
+                if refresh_at > now {
+                    std::thread::sleep(std::time::Duration::from_secs(refresh_at - now));
+                }
+                let result = if let (Some(rt), OAuthFlow::DeviceCode { tenant, client_id }) =
+                    (refresh_token.as_deref(), &flow)
+                {
+                    crate::exchange_ews::oauth::refresh_with_token(
+                        tenant,
+                        client_id,
+                        rt,
+                        allow_invalid_certs,
+                    )
+                } else {
+                    crate::exchange_ews::oauth::acquire(&flow, allow_invalid_certs)
+                };
+                match result {
+                    Ok(tok) => {
+                        client.set_auth(crate::jmap::http::Auth::Bearer {
+                            token: tok.access_token.clone(),
+                        });
+                        if tok.refresh_token.is_some() {
+                            refresh_token = tok.refresh_token.clone();
+                        }
+                        if let Some(new_exp) =
+                            crate::exchange_ews::oauth::decode_jwt_claims(&tok.access_token)
+                                .and_then(|c| c.exp)
+                        {
+                            next_exp = new_exp;
+                        } else {
+                            next_exp = next_exp.saturating_add(50 * 60);
+                        }
+                    }
+                    Err(e) => {
+                        logger.warn(&format!(
+                            "EWS token refresh failed: {e}; sleeping 60s before retry"
+                        ));
+                        std::thread::sleep(std::time::Duration::from_secs(60));
+                    }
+                }
+            }
+        })
+        .ok();
+}
+
+fn run_gc(conn: &Connection) -> Result<(), Error> {
+    let tx = conn.unchecked_transaction()?;
+    db::blobs::gc_orphan_blobs(&tx)?;
+    tx.commit()?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn synthetic_account_id_uses_smtp_for_primary() {
+        assert_eq!(
+            synthetic_account_id("[email protected]", MailboxKind::Primary),
+            "[email protected]"
+        );
+    }
+
+    #[test]
+    fn synthetic_account_id_archive_suffix() {
+        assert_eq!(
+            synthetic_account_id("[email protected]", MailboxKind::Archive),
+            "[email protected]#archive"
+        );
+    }
+
+    #[test]
+    fn synthetic_account_id_public_folders_uses_tenant() {
+        assert_eq!(
+            synthetic_account_id("[email protected]", MailboxKind::PublicFolders),
+            "[email protected]"
+        );
+    }
+
+    #[test]
+    fn basic_against_office365_endpoint_is_refused() {
+        let auth = EwsAuth::Basic {
+            user: "alice@x".to_owned(),
+            password: "p".to_owned(),
+        };
+        let res =
+            enforce_basic_auth_policy(&auth, "https://outlook.office365.com/EWS/Exchange.asmx");
+        assert!(matches!(res, Err(Error::Connection(_))));
+    }
+
+    #[test]
+    fn basic_over_https_on_prem_is_allowed() {
+        let auth = EwsAuth::Basic {
+            user: "alice@x".to_owned(),
+            password: "p".to_owned(),
+        };
+        assert!(
+            enforce_basic_auth_policy(&auth, "https://exchange.example.com/EWS/Exchange.asmx")
+                .is_ok()
+        );
+    }
+
+    #[test]
+    fn basic_over_http_is_refused() {
+        let auth = EwsAuth::Basic {
+            user: "alice".to_owned(),
+            password: "p".to_owned(),
+        };
+        assert!(
+            enforce_basic_auth_policy(&auth, "http://exchange.example.com/EWS/Exchange.asmx")
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn normalise_ews_url_lowercases_host_and_keeps_canonical_path() {
+        assert_eq!(
+            normalise_ews_url("https://OUTLOOK.OFFICE365.COM/EWS/Exchange.asmx"),
+            "https://outlook.office365.com/EWS/Exchange.asmx"
+        );
+        assert_eq!(
+            normalise_ews_url("https://outlook.office365.com:443/ews/exchange.asmx"),
+            "https://outlook.office365.com/EWS/Exchange.asmx"
+        );
+        assert_eq!(
+            normalise_ews_url("https://srv.example.com:8443/EWS/Exchange.asmx"),
+            "https://srv.example.com:8443/EWS/Exchange.asmx"
+        );
+    }
+}

src/sync/import_exchange_ews/folders.rs

@@ -0,0 +1,588 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use rusqlite::{Connection, params};
+
+use crate::db::exchange_ews_ids;
+use crate::error::Error;
+use crate::exchange_ews::EwsClient;
+use crate::exchange_ews::error::EwsError;
+use crate::exchange_ews::parse::{
+    FolderEntry, parse_find_folder_response, parse_folder_inner, parse_get_folder_response,
+    parse_response_messages,
+};
+use crate::exchange_ews::types::{DistinguishedFolderId, FolderClass, MailboxKind};
+use crate::exchange_ews::xml::{
+    FolderRef, FolderShape, Traversal, find_folder_body, get_folder_body,
+};
+use crate::logging::{LEVEL_PROGRESS, Logger};
+use crate::sync::TypeCounts;
+
+pub struct FolderPlan {
+    pub mail: Vec<ClassifiedFolder>,
+    pub calendar: Vec<ClassifiedFolder>,
+    pub contacts: Vec<ClassifiedFolder>,
+    pub well_known_roles: HashMap<String, &'static str>,
+    pub root_folder_id: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassifiedFolder {
+    pub folder: FolderEntry,
+    pub kind: FolderClass,
+}
+
+pub fn plan_folders(
+    client: &EwsClient,
+    url: &str,
+    mailbox_kind: MailboxKind,
+    logger: Logger,
+) -> Result<FolderPlan, EwsError> {
+    let root = mailbox_kind.distinguished_root();
+    let request = find_folder_body(FolderRef::Distinguished(root), Traversal::Deep);
+    let resp = client.call(url, "FindFolder", &request)?;
+    let parsed = parse_find_folder_response(&resp.body)?;
+    if logger.enabled(LEVEL_PROGRESS) {
+        eprintln!("FindFolder returned {} folders", parsed.folders.len());
+    }
+    let mut mail = Vec::new();
+    let mut calendar = Vec::new();
+    let mut contacts = Vec::new();
+    for folder in parsed.folders {
+        let class = FolderClass::from_ipf(&folder.folder_class);
+        match (mailbox_kind, class) {
+            (MailboxKind::Archive, FolderClass::Mail) => {
+                mail.push(ClassifiedFolder {
+                    folder,
+                    kind: class,
+                });
+            }
+            (MailboxKind::Archive, _) => {}
+            (_, FolderClass::Mail) => mail.push(ClassifiedFolder {
+                folder,
+                kind: class,
+            }),
+            (_, FolderClass::Calendar) => {
+                calendar.push(ClassifiedFolder {
+                    folder,
+                    kind: class,
+                });
+            }
+            (_, FolderClass::Contacts) => {
+                contacts.push(ClassifiedFolder {
+                    folder,
+                    kind: class,
+                });
+            }
+            (_, FolderClass::Skipped) => {}
+        }
+    }
+    let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, logger)?;
+    let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind)?;
+    Ok(FolderPlan {
+        mail,
+        calendar,
+        contacts,
+        well_known_roles,
+        root_folder_id,
+    })
+}
+
+fn resolve_well_known_roles(
+    client: &EwsClient,
+    url: &str,
+    mailbox_kind: MailboxKind,
+    logger: Logger,
+) -> Result<HashMap<String, &'static str>, EwsError> {
+    if !matches!(mailbox_kind, MailboxKind::Primary) {
+        return Ok(HashMap::new());
+    }
+    let folders: Vec<FolderRef<'_>> = WELL_KNOWN_ROLES
+        .iter()
+        .map(|(d, _)| FolderRef::Distinguished(*d))
+        .collect();
+    let request = get_folder_body(&folders, FolderShape::IdOnly);
+    let resp = match client.call(url, "GetFolder", &request) {
+        Ok(r) => r,
+        Err(e) => {
+            if logger.enabled(LEVEL_PROGRESS) {
+                eprintln!("GetFolder for distinguished ids failed: {e}; roles will be empty");
+            }
+            return Ok(HashMap::new());
+        }
+    };
+    let messages =
+        parse_response_messages(&resp.body, b"GetFolderResponseMessage").unwrap_or_default();
+    let mut map: HashMap<String, &'static str> = HashMap::new();
+    for (msg, (_, role)) in messages.into_iter().zip(WELL_KNOWN_ROLES.iter()) {
+        if !msg.success {
+            continue;
+        }
+        let Ok(Some(entry)) = parse_folder_inner(&msg.inner_xml) else {
+            continue;
+        };
+        if entry.folder_id.id.is_empty() {
+            continue;
+        }
+        if let Some(r) = role {
+            map.insert(entry.folder_id.id, r);
+        }
+    }
+    Ok(map)
+}
+
+fn resolve_root_folder_id(
+    client: &EwsClient,
+    url: &str,
+    mailbox_kind: MailboxKind,
+) -> Result<String, EwsError> {
+    let root = mailbox_kind.distinguished_root();
+    let body = get_folder_body(&[FolderRef::Distinguished(root)], FolderShape::IdOnly);
+    let resp = client.call(url, "GetFolder", &body)?;
+    let entries = parse_get_folder_response(&resp.body)?;
+    Ok(entries
+        .into_iter()
+        .next()
+        .map(|e| e.folder_id.id)
+        .unwrap_or_default())
+}
+
+const WELL_KNOWN_ROLES: &[(DistinguishedFolderId, Option<&'static str>)] = &[
+    (DistinguishedFolderId::Inbox, Some("inbox")),
+    (DistinguishedFolderId::SentItems, Some("sent")),
+    (DistinguishedFolderId::Drafts, Some("drafts")),
+    (DistinguishedFolderId::DeletedItems, Some("trash")),
+    (DistinguishedFolderId::JunkEmail, Some("junk")),
+    (DistinguishedFolderId::Archive, Some("archive")),
+    (DistinguishedFolderId::Outbox, None),
+    (DistinguishedFolderId::ConversationHistory, None),
+];
+
+pub fn reconcile(
+    conn: &mut Connection,
+    source_id: i64,
+    plan: &FolderPlan,
+    mailbox_counts: &mut TypeCounts,
+    calendar_counts: &mut TypeCounts,
+    addressbook_counts: &mut TypeCounts,
+    _logger: Logger,
+) -> Result<(), Error> {
+    let local_mailbox: HashMap<String, exchange_ews_ids::FolderRow> =
+        exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::MAILBOX)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let local_calendar: HashMap<String, exchange_ews_ids::FolderRow> =
+        exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::CALENDAR)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let local_addressbook: HashMap<String, exchange_ews_ids::FolderRow> =
+        exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::ADDRESS_BOOK)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+
+    let mut id_to_local: HashMap<String, i64> = HashMap::new();
+    for (id, row) in local_mailbox
+        .iter()
+        .chain(local_calendar.iter())
+        .chain(local_addressbook.iter())
+    {
+        id_to_local.insert(id.clone(), row.local_id);
+    }
+
+    let ordered_mail = order_by_parent(&plan.mail);
+    let ordered_calendar = order_by_parent(&plan.calendar);
+    let ordered_contacts = order_by_parent(&plan.contacts);
+
+    let mut server_ids_mail: Vec<String> = Vec::new();
+    let mut server_ids_calendar: Vec<String> = Vec::new();
+    let mut server_ids_addressbook: Vec<String> = Vec::new();
+
+    for folder in &ordered_mail {
+        server_ids_mail.push(folder.folder.folder_id.id.clone());
+        let role = plan
+            .well_known_roles
+            .get(&folder.folder.folder_id.id)
+            .copied();
+        let mut ctx = UpsertCtx {
+            conn,
+            source_id,
+            id_to_local: &mut id_to_local,
+            local: &local_mailbox,
+            counts: mailbox_counts,
+            root_folder_id: &plan.root_folder_id,
+        };
+        upsert_mailbox(&mut ctx, folder, role)?;
+    }
+    for folder in &ordered_calendar {
+        server_ids_calendar.push(folder.folder.folder_id.id.clone());
+        let mut ctx = UpsertCtx {
+            conn,
+            source_id,
+            id_to_local: &mut id_to_local,
+            local: &local_calendar,
+            counts: calendar_counts,
+            root_folder_id: &plan.root_folder_id,
+        };
+        upsert_calendar(&mut ctx, folder)?;
+    }
+    for folder in &ordered_contacts {
+        server_ids_addressbook.push(folder.folder.folder_id.id.clone());
+        let mut ctx = UpsertCtx {
+            conn,
+            source_id,
+            id_to_local: &mut id_to_local,
+            local: &local_addressbook,
+            counts: addressbook_counts,
+            root_folder_id: &plan.root_folder_id,
+        };
+        upsert_address_book(&mut ctx, folder)?;
+    }
+
+    delete_vanished(
+        conn,
+        source_id,
+        exchange_ews_ids::MAILBOX,
+        "mailboxes",
+        &local_mailbox,
+        &server_ids_mail,
+        mailbox_counts,
+    )?;
+    delete_vanished(
+        conn,
+        source_id,
+        exchange_ews_ids::CALENDAR,
+        "calendars",
+        &local_calendar,
+        &server_ids_calendar,
+        calendar_counts,
+    )?;
+    delete_vanished(
+        conn,
+        source_id,
+        exchange_ews_ids::ADDRESS_BOOK,
+        "address_books",
+        &local_addressbook,
+        &server_ids_addressbook,
+        addressbook_counts,
+    )?;
+    Ok(())
+}
+
+fn order_by_parent(folders: &[ClassifiedFolder]) -> Vec<&ClassifiedFolder> {
+    let by_id: HashMap<&str, &ClassifiedFolder> = folders
+        .iter()
+        .map(|f| (f.folder.folder_id.id.as_str(), f))
+        .collect();
+    let mut depths: HashMap<&str, usize> = HashMap::new();
+    fn depth_of<'a>(
+        id: &'a str,
+        by_id: &HashMap<&'a str, &'a ClassifiedFolder>,
+        memo: &mut HashMap<&'a str, usize>,
+        seen: &mut std::collections::HashSet<&'a str>,
+    ) -> usize {
+        if let Some(d) = memo.get(id) {
+            return *d;
+        }
+        if !seen.insert(id) {
+            return 0;
+        }
+        let d = match by_id.get(id).and_then(|f| f.folder.parent_id.as_deref()) {
+            Some(parent) if by_id.contains_key(parent) => 1 + depth_of(parent, by_id, memo, seen),
+            _ => 0,
+        };
+        memo.insert(id, d);
+        d
+    }
+    for f in folders {
+        let id = f.folder.folder_id.id.as_str();
+        let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
+        let d = depth_of(id, &by_id, &mut depths, &mut seen);
+        depths.insert(id, d);
+    }
+    let mut out: Vec<&ClassifiedFolder> = folders.iter().collect();
+    out.sort_by_key(|f| {
+        depths
+            .get(f.folder.folder_id.id.as_str())
+            .copied()
+            .unwrap_or(0)
+    });
+    out
+}
+
+struct UpsertCtx<'a> {
+    conn: &'a mut Connection,
+    source_id: i64,
+    id_to_local: &'a mut HashMap<String, i64>,
+    local: &'a HashMap<String, exchange_ews_ids::FolderRow>,
+    counts: &'a mut TypeCounts,
+    root_folder_id: &'a str,
+}
+
+fn upsert_mailbox(
+    ctx: &mut UpsertCtx<'_>,
+    folder: &ClassifiedFolder,
+    role: Option<&'static str>,
+) -> Result<(), Error> {
+    let parent_local = parent_local_id(folder, ctx.id_to_local, ctx.root_folder_id);
+    let tx = ctx
+        .conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let existing = ctx.local.get(&folder.folder.folder_id.id);
+    let local_id = if let Some(row) = existing {
+        tx.execute(
+            "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3 WHERE id = ?4",
+            params![folder.folder.display_name, parent_local, role, row.local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        exchange_ews_ids::update_change_key(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.fetched += 1;
+        row.local_id
+    } else {
+        tx.execute(
+            "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) \
+             VALUES (?1, ?2, ?3, 0, 1)",
+            params![folder.folder.display_name, parent_local, role],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let new_id = tx.last_insert_rowid();
+        exchange_ews_ids::insert(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            folder.folder.parent_id.as_deref().unwrap_or(""),
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+            new_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.created += 1;
+        new_id
+    };
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    ctx.id_to_local
+        .insert(folder.folder.folder_id.id.clone(), local_id);
+    Ok(())
+}
+
+fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> {
+    let tx = ctx
+        .conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let existing = ctx.local.get(&folder.folder.folder_id.id);
+    let local_id = if let Some(row) = existing {
+        tx.execute(
+            "UPDATE calendars SET name = ?1 WHERE id = ?2",
+            params![folder.folder.display_name, row.local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        exchange_ews_ids::update_change_key(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.fetched += 1;
+        row.local_id
+    } else {
+        tx.execute(
+            "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible) \
+             VALUES (?1, 0, 1, 1)",
+            params![folder.folder.display_name],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let new_id = tx.last_insert_rowid();
+        exchange_ews_ids::insert(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::CALENDAR,
+            folder.folder.parent_id.as_deref().unwrap_or(""),
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+            new_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.created += 1;
+        new_id
+    };
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    ctx.id_to_local
+        .insert(folder.folder.folder_id.id.clone(), local_id);
+    Ok(())
+}
+
+fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> {
+    let tx = ctx
+        .conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let existing = ctx.local.get(&folder.folder.folder_id.id);
+    let local_id = if let Some(row) = existing {
+        tx.execute(
+            "UPDATE address_books SET name = ?1 WHERE id = ?2",
+            params![folder.folder.display_name, row.local_id],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        exchange_ews_ids::update_change_key(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.fetched += 1;
+        row.local_id
+    } else {
+        tx.execute(
+            "INSERT INTO address_books (name, sort_order, is_subscribed) \
+             VALUES (?1, 0, 1)",
+            params![folder.folder.display_name],
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        let new_id = tx.last_insert_rowid();
+        exchange_ews_ids::insert(
+            &tx,
+            ctx.source_id,
+            exchange_ews_ids::ADDRESS_BOOK,
+            folder.folder.parent_id.as_deref().unwrap_or(""),
+            &folder.folder.folder_id.id,
+            &folder.folder.folder_id.change_key,
+            new_id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+        ctx.counts.created += 1;
+        new_id
+    };
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    ctx.id_to_local
+        .insert(folder.folder.folder_id.id.clone(), local_id);
+    Ok(())
+}
+
+fn parent_local_id(
+    folder: &ClassifiedFolder,
+    id_to_local: &HashMap<String, i64>,
+    root_folder_id: &str,
+) -> Option<i64> {
+    let parent_id = folder.folder.parent_id.as_deref()?;
+    if parent_id == root_folder_id || parent_id.is_empty() {
+        return None;
+    }
+    id_to_local.get(parent_id).copied()
+}
+
+fn delete_vanished(
+    conn: &mut Connection,
+    source_id: i64,
+    type_name: &str,
+    table: &str,
+    local: &HashMap<String, exchange_ews_ids::FolderRow>,
+    server_ids: &[String],
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let server_set: std::collections::HashSet<&str> =
+        server_ids.iter().map(String::as_str).collect();
+    let mut vanished: Vec<(&str, &exchange_ews_ids::FolderRow)> = local
+        .iter()
+        .filter(|(id, _)| !server_set.contains(id.as_str()))
+        .map(|(id, row)| (id.as_str(), row))
+        .collect();
+    if type_name == exchange_ews_ids::MAILBOX {
+        vanished.sort_by_key(|(_, row)| std::cmp::Reverse(folder_depth(row, local)));
+    }
+    for (item_id, row) in vanished {
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        let result = tx.execute(
+            &format!("DELETE FROM {table} WHERE id = ?1"),
+            params![row.local_id],
+        );
+        match result {
+            Ok(_) => {
+                exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id)
+                    .map_err(|e| Error::Partial(e.to_string()))?;
+                tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                counts.deleted += 1;
+            }
+            Err(_) => {
+                let _ = tx.rollback();
+                counts.failed += 1;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn folder_depth(
+    row: &exchange_ews_ids::FolderRow,
+    local: &HashMap<String, exchange_ews_ids::FolderRow>,
+) -> usize {
+    let mut depth = 0;
+    let mut cursor = row.folder_id.as_str();
+    let mut visited: std::collections::HashSet<&str> = std::collections::HashSet::new();
+    while !cursor.is_empty() && visited.insert(cursor) {
+        match local.get(cursor) {
+            Some(parent) => {
+                depth += 1;
+                cursor = parent.folder_id.as_str();
+            }
+            None => break,
+        }
+    }
+    depth
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn row(id: &str, parent_ews_id: &str, local_id: i64) -> exchange_ews_ids::FolderRow {
+        exchange_ews_ids::FolderRow {
+            item_id: id.to_owned(),
+            folder_id: parent_ews_id.to_owned(),
+            change_key: String::new(),
+            local_id,
+        }
+    }
+
+    #[test]
+    fn folder_depth_walks_parent_chain() {
+        let mut local: HashMap<String, exchange_ews_ids::FolderRow> = HashMap::new();
+        local.insert("ROOT".to_owned(), row("ROOT", "", 1));
+        local.insert("CHILD".to_owned(), row("CHILD", "ROOT", 2));
+        local.insert("GRAND".to_owned(), row("GRAND", "CHILD", 3));
+        assert_eq!(folder_depth(local.get("ROOT").unwrap(), &local), 0);
+        assert_eq!(folder_depth(local.get("CHILD").unwrap(), &local), 1);
+        assert_eq!(folder_depth(local.get("GRAND").unwrap(), &local), 2);
+    }
+
+    #[test]
+    fn folder_depth_handles_unknown_parent() {
+        let mut local: HashMap<String, exchange_ews_ids::FolderRow> = HashMap::new();
+        local.insert("X".to_owned(), row("X", "ORPHAN", 1));
+        assert_eq!(folder_depth(local.get("X").unwrap(), &local), 0);
+    }
+
+    #[test]
+    fn folder_depth_stops_on_cycle() {
+        let mut local: HashMap<String, exchange_ews_ids::FolderRow> = HashMap::new();
+        local.insert("A".to_owned(), row("A", "B", 1));
+        local.insert("B".to_owned(), row("B", "A", 2));
+        let d = folder_depth(local.get("A").unwrap(), &local);
+        assert!(d <= 2);
+    }
+}

src/sync/import_exchange_ews/items.rs

@@ -0,0 +1,469 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use std::collections::HashMap;
+
+use crate::error::Error;
+use crate::exchange_ews::EwsClient;
+use crate::exchange_ews::error::EwsError;
+use crate::exchange_ews::parse::{
+    SyncChange, parse_find_item_response, parse_response_messages, parse_sync_folder_items_response,
+};
+use crate::exchange_ews::types::{FolderId, ItemId, ResponseCode};
+use crate::exchange_ews::xml::{
+    FolderRef, ItemShape, Traversal, find_item_body, get_item_body, sync_folder_items_body,
+};
+use crate::logging::Logger;
+
+pub struct ItemRunCtx<'a> {
+    pub client: &'a EwsClient,
+    pub url: &'a str,
+    pub source_id: i64,
+    pub batch_size: usize,
+    pub attachment_batch: usize,
+    pub connections: usize,
+    pub use_syncfolderitems: bool,
+    pub logger: Logger,
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumeratedItem {
+    pub element: String,
+    pub id: ItemId,
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumerationOutcome {
+    pub items: Vec<EnumeratedItem>,
+    pub mode: EnumerationMode,
+}
+
+#[derive(Debug, Clone)]
+pub enum EnumerationMode {
+    Full,
+    Delta {
+        deletions: Vec<String>,
+        new_sync_state: String,
+    },
+}
+
+pub fn enumerate_folder(
+    ctx: &ItemRunCtx<'_>,
+    folder: &FolderId,
+    prior_sync_state: Option<&str>,
+) -> Result<EnumerationOutcome, EwsError> {
+    if ctx.use_syncfolderitems
+        && let Some(state) = prior_sync_state
+        && !state.is_empty()
+        && let Some(outcome) = try_sync_folder_items(ctx, folder, state)?
+    {
+        return Ok(outcome);
+    }
+    enumerate_via_find_item(ctx, folder).map(|items| EnumerationOutcome {
+        items,
+        mode: EnumerationMode::Full,
+    })
+}
+
+fn sync_fallback_codes(code: &ResponseCode) -> bool {
+    matches!(
+        code,
+        ResponseCode::InvalidIdMalformed | ResponseCode::AccessDenied | ResponseCode::Other(_)
+    )
+}
+
+fn try_sync_folder_items(
+    ctx: &ItemRunCtx<'_>,
+    folder: &FolderId,
+    prior_state: &str,
+) -> Result<Option<EnumerationOutcome>, EwsError> {
+    let mut sync_state = prior_state.to_owned();
+    let mut items: Vec<EnumeratedItem> = Vec::new();
+    let mut deletions: Vec<String> = Vec::new();
+    let mut iters = 0;
+    loop {
+        let body = sync_folder_items_body(folder, &sync_state, 512);
+        let resp = match ctx.client.call(ctx.url, "SyncFolderItems", &body) {
+            Ok(r) => r,
+            Err(EwsError::SoapFault {
+                code: ResponseCode::InvalidSyncStateData,
+                ..
+            }) => {
+                return Ok(None);
+            }
+            Err(EwsError::SoapFault { code, .. }) if sync_fallback_codes(&code) => {
+                return Ok(None);
+            }
+            Err(e) => return Err(e),
+        };
+        let parsed = parse_sync_folder_items_response(&resp.body)?;
+        for change in parsed.changes {
+            match change {
+                SyncChange::Create { id, element } | SyncChange::Update { id, element } => {
+                    items.push(EnumeratedItem { element, id });
+                }
+                SyncChange::Delete { id } => {
+                    deletions.push(id.id);
+                }
+                SyncChange::ReadFlagChange { id, .. } => {
+                    items.push(EnumeratedItem {
+                        element: "Message".to_owned(),
+                        id,
+                    });
+                }
+            }
+        }
+        sync_state = parsed.sync_state;
+        if !parsed.more {
+            break;
+        }
+        iters += 1;
+        if iters > 200 {
+            break;
+        }
+    }
+    Ok(Some(EnumerationOutcome {
+        items,
+        mode: EnumerationMode::Delta {
+            deletions,
+            new_sync_state: sync_state,
+        },
+    }))
+}
+
+fn enumerate_via_find_item(
+    ctx: &ItemRunCtx<'_>,
+    folder: &FolderId,
+) -> Result<Vec<EnumeratedItem>, EwsError> {
+    let mut items: Vec<EnumeratedItem> = Vec::new();
+    let mut offset: u32 = 0;
+    let page_size: u32 = 500;
+    loop {
+        let body = find_item_body(
+            FolderRef::Concrete(folder),
+            Traversal::Shallow,
+            offset,
+            page_size,
+        );
+        let resp = ctx.client.call(ctx.url, "FindItem", &body)?;
+        let parsed = parse_find_item_response(&resp.body)?;
+        let returned = parsed.items.len() as u32;
+        for entry in parsed.items {
+            items.push(EnumeratedItem {
+                element: entry.element,
+                id: entry.id,
+            });
+        }
+        if !parsed.more {
+            break;
+        }
+        if returned == 0 {
+            break;
+        }
+        offset = offset.saturating_add(page_size);
+        if offset > 1_000_000 {
+            break;
+        }
+    }
+    Ok(items)
+}
+
+#[derive(Debug, Clone)]
+pub struct DiffPlan {
+    pub new: Vec<ItemId>,
+    pub vanished: Vec<(String, i64)>,
+    pub present_changed: Vec<(ItemId, i64)>,
+    pub present_unchanged: Vec<(String, i64)>,
+}
+
+pub fn plan_for(
+    outcome: &EnumerationOutcome,
+    local: &[crate::db::exchange_ews_ids::ItemRow],
+) -> DiffPlan {
+    match &outcome.mode {
+        EnumerationMode::Full => diff(&outcome.items, local),
+        EnumerationMode::Delta { deletions, .. } => diff_delta(&outcome.items, deletions, local),
+    }
+}
+
+fn diff_delta(
+    server_changes: &[EnumeratedItem],
+    deletions: &[String],
+    local: &[crate::db::exchange_ews_ids::ItemRow],
+) -> DiffPlan {
+    let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> =
+        local.iter().map(|r| (r.item_id.as_str(), r)).collect();
+    let mut plan = DiffPlan {
+        new: Vec::new(),
+        vanished: Vec::new(),
+        present_changed: Vec::new(),
+        present_unchanged: Vec::new(),
+    };
+    for s in server_changes {
+        match local_map.get(s.id.id.as_str()) {
+            None => plan.new.push(s.id.clone()),
+            Some(row) => plan.present_changed.push((s.id.clone(), row.local_id)),
+        }
+    }
+    for d in deletions {
+        if let Some(row) = local_map.get(d.as_str()) {
+            plan.vanished.push((d.clone(), row.local_id));
+        }
+    }
+    plan
+}
+
+pub fn diff(server: &[EnumeratedItem], local: &[crate::db::exchange_ews_ids::ItemRow]) -> DiffPlan {
+    let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> =
+        local.iter().map(|r| (r.item_id.as_str(), r)).collect();
+    let server_map: HashMap<&str, &EnumeratedItem> =
+        server.iter().map(|s| (s.id.id.as_str(), s)).collect();
+    let mut plan = DiffPlan {
+        new: Vec::new(),
+        vanished: Vec::new(),
+        present_changed: Vec::new(),
+        present_unchanged: Vec::new(),
+    };
+    for s in server {
+        match local_map.get(s.id.id.as_str()) {
+            None => plan.new.push(s.id.clone()),
+            Some(row) => {
+                if s.id.change_key.is_empty() || s.id.change_key == row.change_key {
+                    plan.present_unchanged
+                        .push((row.item_id.clone(), row.local_id));
+                } else {
+                    plan.present_changed.push((s.id.clone(), row.local_id));
+                }
+            }
+        }
+    }
+    for row in local {
+        if !server_map.contains_key(row.item_id.as_str()) {
+            plan.vanished.push((row.item_id.clone(), row.local_id));
+        }
+    }
+    plan
+}
+
+pub struct GetItemBatchOutcome {
+    pub messages: Vec<crate::exchange_ews::parse::ResponseMessage>,
+    pub failed_items: u64,
+}
+
+fn is_per_batch_fault(err: &EwsError) -> bool {
+    matches!(
+        err,
+        EwsError::SoapFault { .. } | EwsError::HttpStatus { .. } | EwsError::Malformed(_)
+    )
+}
+
+pub fn get_items(
+    ctx: &ItemRunCtx<'_>,
+    shape: ItemShape,
+    ids: &[ItemId],
+) -> Result<GetItemBatchOutcome, EwsError> {
+    let batch = ctx.batch_size.max(1);
+    let workers = ctx.connections.clamp(1, 8);
+    let mut failed_items: u64 = 0;
+    if workers <= 1 || ids.len() <= batch {
+        let mut all = Vec::new();
+        for chunk in ids.chunks(batch) {
+            let body = get_item_body(shape, chunk);
+            match ctx.client.call(ctx.url, "GetItem", &body) {
+                Ok(resp) => match parse_response_messages(&resp.body, b"GetItemResponseMessage") {
+                    Ok(mut msgs) => all.append(&mut msgs),
+                    Err(e) if is_per_batch_fault(&e) => {
+                        ctx.logger.warn(&format!(
+                            "GetItem batch parse failed ({e}); {} ids left for retry next run",
+                            chunk.len()
+                        ));
+                        failed_items += chunk.len() as u64;
+                    }
+                    Err(e) => return Err(e),
+                },
+                Err(e) if is_per_batch_fault(&e) => {
+                    ctx.logger.warn(&format!(
+                        "GetItem batch failed ({e}); {} ids left for retry next run",
+                        chunk.len()
+                    ));
+                    failed_items += chunk.len() as u64;
+                }
+                Err(e) => return Err(e),
+            }
+        }
+        return Ok(GetItemBatchOutcome {
+            messages: all,
+            failed_items,
+        });
+    }
+    let client = ctx.client.clone();
+    let url = ctx.url.to_owned();
+    type BatchResult = (
+        usize,
+        Result<Vec<crate::exchange_ews::parse::ResponseMessage>, EwsError>,
+    );
+    let pool: crate::sync::import_jmap::pool::Pool<Vec<ItemId>, BatchResult> =
+        crate::sync::import_jmap::pool::Pool::new(workers, move |chunk: Vec<ItemId>| {
+            let body = get_item_body(shape, &chunk);
+            let n = chunk.len();
+            let result = match client.call(&url, "GetItem", &body) {
+                Ok(resp) => parse_response_messages(&resp.body, b"GetItemResponseMessage"),
+                Err(e) => Err(e),
+            };
+            (n, result)
+        });
+    let mut submitted = 0usize;
+    for chunk in ids.chunks(batch) {
+        pool.submit(chunk.to_vec());
+        submitted += 1;
+    }
+    let mut all = Vec::new();
+    let mut abort_err: Option<EwsError> = None;
+    for _ in 0..submitted {
+        match pool.results().recv() {
+            Ok((_, Ok(mut msgs))) => all.append(&mut msgs),
+            Ok((n, Err(e))) => {
+                if is_per_batch_fault(&e) {
+                    ctx.logger.warn(&format!(
+                        "GetItem batch failed ({e}); {n} ids left for retry next run"
+                    ));
+                    failed_items += n as u64;
+                } else if abort_err.is_none() {
+                    abort_err = Some(e);
+                }
+            }
+            Err(_) => break,
+        }
+    }
+    if let Some(e) = abort_err {
+        return Err(e);
+    }
+    Ok(GetItemBatchOutcome {
+        messages: all,
+        failed_items,
+    })
+}
+
+pub fn delete_vanished(
+    conn: &mut rusqlite::Connection,
+    source_id: i64,
+    type_name: &str,
+    table: &str,
+    vanished: &[(String, i64)],
+    counts: &mut crate::sync::TypeCounts,
+) -> Result<(), Error> {
+    use rusqlite::params;
+    for (item_id, local_id) in vanished {
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| Error::Partial(e.to_string()))?;
+        match tx.execute(
+            &format!("DELETE FROM {table} WHERE id = ?1"),
+            params![local_id],
+        ) {
+            Ok(_) => {
+                crate::db::exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id)
+                    .map_err(|e| Error::Partial(e.to_string()))?;
+                tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+                counts.deleted += 1;
+            }
+            Err(_) => {
+                let _ = tx.rollback();
+                counts.failed += 1;
+            }
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn diff_delta_only_yields_changes_from_server_response() {
+        use crate::db::exchange_ews_ids::ItemRow;
+        let outcome = EnumerationOutcome {
+            items: vec![EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("A", "ck-2"),
+            }],
+            mode: EnumerationMode::Delta {
+                deletions: vec!["Z".to_owned()],
+                new_sync_state: "STATE2".to_owned(),
+            },
+        };
+        let local = vec![
+            ItemRow {
+                item_id: "A".to_owned(),
+                change_key: "ck-1".to_owned(),
+                local_id: 1,
+            },
+            ItemRow {
+                item_id: "B".to_owned(),
+                change_key: "ck-1".to_owned(),
+                local_id: 2,
+            },
+            ItemRow {
+                item_id: "Z".to_owned(),
+                change_key: "ck-9".to_owned(),
+                local_id: 99,
+            },
+        ];
+        let plan = plan_for(&outcome, &local);
+        assert!(plan.new.is_empty());
+        assert_eq!(plan.present_changed.len(), 1);
+        assert_eq!(plan.present_changed[0].0.id, "A");
+        assert_eq!(plan.vanished.len(), 1);
+        assert_eq!(plan.vanished[0].0, "Z");
+        assert_eq!(plan.present_unchanged.len(), 0);
+    }
+
+    #[test]
+    fn diff_splits_into_new_unchanged_changed_vanished() {
+        use crate::db::exchange_ews_ids::ItemRow;
+        let server = vec![
+            EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("A", "ck-1"),
+            },
+            EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("B", "ck-1"),
+            },
+            EnumeratedItem {
+                element: "Message".to_owned(),
+                id: ItemId::new("C", "ck-2"),
+            },
+        ];
+        let local = vec![
+            ItemRow {
+                item_id: "A".to_owned(),
+                change_key: "ck-1".to_owned(),
+                local_id: 1,
+            },
+            ItemRow {
+                item_id: "B".to_owned(),
+                change_key: "ck-0".to_owned(),
+                local_id: 2,
+            },
+            ItemRow {
+                item_id: "Z".to_owned(),
+                change_key: "ck-9".to_owned(),
+                local_id: 99,
+            },
+        ];
+        let plan = diff(&server, &local);
+        assert_eq!(plan.new.len(), 1);
+        assert_eq!(plan.new[0].id, "C");
+        assert_eq!(plan.present_unchanged.len(), 1);
+        assert_eq!(plan.present_unchanged[0].0, "A");
+        assert_eq!(plan.present_changed.len(), 1);
+        assert_eq!(plan.present_changed[0].0.id, "B");
+        assert_eq!(plan.vanished.len(), 1);
+        assert_eq!(plan.vanished[0].0, "Z");
+    }
+}

src/sync/import_exchange_ews/messages.rs

@@ -0,0 +1,262 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD;
+use rusqlite::{Connection, params};
+use serde_json::{Value, json};
+
+use crate::db::{blobs, exchange_ews_ids};
+use crate::error::Error;
+use crate::exchange_ews::parse::{MessageItem, parse_message_item};
+use crate::exchange_ews::types::ItemId;
+use crate::exchange_ews::xml::ItemShape;
+use crate::logging::LEVEL_PROGRESS;
+use crate::sync::TypeCounts;
+use crate::sync::emailmeta::email_meta_from_blob;
+use crate::sync::keys::index_to_json;
+
+use super::folders::FolderPlan;
+use super::items::{
+    EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
+};
+
+pub fn reconcile_all(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    plan: &FolderPlan,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    for folder in &plan.mail {
+        let folder_id = &folder.folder.folder_id;
+        let local_folder_id = match exchange_ews_ids::local_for_item(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            &folder_id.id,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?
+        {
+            Some(id) => id,
+            None => continue,
+        };
+        if let Err(e) = reconcile_one_folder(conn, ctx, folder_id, local_folder_id, counts) {
+            ctx.logger
+                .warn(&format!("email folder {} failed: {}", folder_id.id, e));
+            counts.failed += 1;
+        }
+    }
+    Ok(())
+}
+
+fn reconcile_one_folder(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    folder: &crate::exchange_ews::types::FolderId,
+    local_folder_id: i64,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let prior = exchange_ews_ids::get_sync_state(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::MAILBOX,
+        &folder.id,
+    )
+    .map_err(|e| Error::Partial(e.to_string()))?;
+    let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?;
+    let local =
+        exchange_ews_ids::items_in_folder(conn, ctx.source_id, exchange_ews_ids::EMAIL, &folder.id)
+            .map_err(|e| Error::Partial(e.to_string()))?;
+    let plan = plan_for(&outcome, &local);
+    if ctx.logger.enabled(LEVEL_PROGRESS) {
+        eprintln!(
+            "EWS folder {}: new={} changed={} vanished={} unchanged={}",
+            folder.id,
+            plan.new.len(),
+            plan.present_changed.len(),
+            plan.vanished.len(),
+            plan.present_unchanged.len()
+        );
+    }
+    let mut to_fetch: Vec<ItemId> = plan.new.clone();
+    for (id, _local_id) in &plan.present_changed {
+        to_fetch.push(id.clone());
+    }
+    if !to_fetch.is_empty() {
+        let outcome = get_items(ctx, ItemShape::Message, &to_fetch).map_err(Error::from)?;
+        counts.failed += outcome.failed_items;
+        for msg in outcome.messages {
+            if !msg.success {
+                if matches!(
+                    msg.response_code,
+                    crate::exchange_ews::types::ResponseCode::ItemNotFound
+                ) {
+                    counts.skipped += 1;
+                } else {
+                    counts.failed += 1;
+                    ctx.logger.warn(&format!(
+                        "GetItem (message) error: {} {}",
+                        msg.response_code, msg.message_text
+                    ));
+                }
+                continue;
+            }
+            let parsed = parse_message_item(&msg.inner_xml).map_err(Error::from)?;
+            if parsed.id.id.is_empty() {
+                counts.failed += 1;
+                continue;
+            }
+            let existing = plan
+                .present_changed
+                .iter()
+                .find(|(id, _)| id.id == parsed.id.id)
+                .map(|(_, local)| *local);
+            apply_message(
+                conn,
+                ctx,
+                &parsed,
+                local_folder_id,
+                &folder.id,
+                existing,
+                counts,
+            )?;
+        }
+    }
+    delete_vanished(
+        conn,
+        ctx.source_id,
+        exchange_ews_ids::EMAIL,
+        "emails",
+        &plan.vanished,
+        counts,
+    )?;
+    if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode {
+        exchange_ews_ids::set_sync_state(
+            conn,
+            ctx.source_id,
+            exchange_ews_ids::MAILBOX,
+            &folder.id,
+            new_sync_state,
+        )
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    }
+    Ok(())
+}
+
+fn apply_message(
+    conn: &mut Connection,
+    ctx: &ItemRunCtx<'_>,
+    item: &MessageItem,
+    local_folder_id: i64,
+    folder_id: &str,
+    existing_local_id: Option<i64>,
+    counts: &mut TypeCounts,
+) -> Result<(), Error> {
+    let mime_b64 = match item.mime_content.as_ref() {
+        Some(s) => s.replace(['\n', '\r', ' ', '\t'], ""),
+        None => {
+            counts.skipped += 1;
+            ctx.logger.warn(&format!(
+                "message {} has no MimeContent; skipping",
+                item.id.id
+            ));
+            return Ok(());
+        }
+    };
+    let bytes = match STANDARD.decode(mime_b64.as_bytes()) {
+        Ok(b) => b,
+        Err(e) => {
+            counts.failed += 1;
+            ctx.logger.warn(&format!(
+                "message {} MimeContent base64 decode failed: {e}",
+                item.id.id
+            ));
+            return Ok(());
+        }
+    };
+    let (idx, _date_header) = email_meta_from_blob(&bytes);
+    let message_match = index_to_json(&idx);
+    let received_at = item
+        .date_time_received
+        .clone()
+        .unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned());
+    let keywords = keyword_array(item);
+    let mailbox_ids = json!([local_folder_id]).to_string();
+
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| Error::Partial(e.to_string()))?;
+    let blob_id = blobs::intern_blob(&tx, &bytes).map_err(|e| Error::Partial(e.to_string()))?;
+    match existing_local_id {
+        Some(id) => {
+            tx.execute(
+                "UPDATE emails SET blob_id = ?1, received_at = ?2, mailbox_ids = ?3, \
+                 keywords = ?4, message_match = ?5 WHERE id = ?6",
+                params![
+                    blob_id,
+                    received_at,
+                    mailbox_ids,
+                    keywords,
+                    message_match,
+                    id
+                ],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            exchange_ews_ids::update_change_key(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::EMAIL,
+                &item.id.id,
+                &item.id.change_key,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.fetched += 1;
+        }
+        None => {
+            tx.execute(
+                "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) \
+                 VALUES (?1, ?2, ?3, ?4, ?5)",
+                params![blob_id, received_at, mailbox_ids, keywords, message_match],
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            let new_id = tx.last_insert_rowid();
+            exchange_ews_ids::insert(
+                &tx,
+                ctx.source_id,
+                exchange_ews_ids::EMAIL,
+                folder_id,
+                &item.id.id,
+                &item.id.change_key,
+                new_id,
+            )
+            .map_err(|e| Error::Partial(e.to_string()))?;
+            counts.created += 1;
+        }
+    }
+    tx.commit().map_err(|e| Error::Partial(e.to_string()))?;
+    Ok(())
+}
+
+fn keyword_array(item: &MessageItem) -> String {
+    let mut kws: Vec<String> = Vec::new();
+    if matches!(item.is_read, Some(true)) {
+        kws.push("$seen".to_owned());
+    }
+    if matches!(item.is_draft, Some(true)) {
+        kws.push("$draft".to_owned());
+    }
+    if matches!(item.is_read_receipt_requested, Some(true)) {
+        kws.push("$notified".to_owned());
+    }
+    if matches!(item.flag_status.as_deref(), Some("Flagged")) {
+        kws.push("$flagged".to_owned());
+    }
+    for cat in &item.categories {
+        kws.push(cat.to_ascii_lowercase());
+    }
+    let value: Value = Value::Array(kws.into_iter().map(Value::String).collect());
+    value.to_string()
+}