diff --git a/CHANGELOG.md b/CHANGELOG.md index c82f09f..52b2696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.1] - 2026-06-XX + +### Added + +### Changed + +### Fixed +- MS Exchange duplicate ids and incorrect JSCalendar mapping issues. + ## [1.0.0] - 2026-05-29 ### Added diff --git a/Cargo.lock b/Cargo.lock index 8ae62e4..30e3dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2937,7 +2937,7 @@ dependencies = [ [[package]] name = "vandelay" -version = "1.0.0" +version = "1.0.1" dependencies = [ "base64", "blake3", diff --git a/Cargo.toml b/Cargo.toml index b425262..be6cae2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "vandelay" description = "JMAP account migration utility" -version = "1.0.0" +version = "1.0.1" authors = ["Stalwart Labs LLC "] license = "Apache-2.0 OR MIT" repository = "https://github.com/stalwartlabs/vandelay" diff --git a/src/sync/import_exchange_graph/calendar.rs b/src/sync/import_exchange_graph/calendar.rs index 02f0923..9f4981e 100644 --- a/src/sync/import_exchange_graph/calendar.rs +++ b/src/sync/import_exchange_graph/calendar.rs @@ -33,6 +33,7 @@ pub fn reconcile_all( let local: HashMap = exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?; let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); + let mut planned: std::collections::HashSet = std::collections::HashSet::new(); let mut any_failure = false; for cal in calendars { @@ -55,7 +56,7 @@ pub fn reconcile_all( _ => { if let Some(id) = stub.get("id").and_then(Value::as_str) { server_total.insert(id.to_owned()); - if !local.contains_key(id) { + if !local.contains_key(id) && planned.insert(id.to_owned()) { want_ids.push(id.to_owned()); } } diff --git a/src/sync/import_exchange_graph/contacts.rs b/src/sync/import_exchange_graph/contacts.rs index 1c46b6e..fa0c5da 100644 --- a/src/sync/import_exchange_graph/contacts.rs +++ b/src/sync/import_exchange_graph/contacts.rs @@ -30,6 +30,7 @@ pub fn reconcile_all( let local: HashMap = exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?; let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); + let mut planned: std::collections::HashSet = std::collections::HashSet::new(); let mut any_failure = false; for book in books { @@ -60,7 +61,7 @@ pub fn reconcile_all( } let new_ids: Vec = ids .into_iter() - .filter(|id| !local.contains_key(id)) + .filter(|id| !local.contains_key(id) && planned.insert(id.clone())) .collect(); if new_ids.is_empty() { continue; diff --git a/src/sync/import_exchange_graph/messages.rs b/src/sync/import_exchange_graph/messages.rs index 9b136a9..f23bbd9 100644 --- a/src/sync/import_exchange_graph/messages.rs +++ b/src/sync/import_exchange_graph/messages.rs @@ -30,6 +30,7 @@ pub fn reconcile_all( let local: std::collections::HashMap = exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?; let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); + let mut planned: std::collections::HashSet = std::collections::HashSet::new(); let mut any_failure = false; for folder in folders { @@ -58,7 +59,7 @@ pub fn reconcile_all( } let new_ids: Vec = ids .into_iter() - .filter(|id| !local.contains_key(id)) + .filter(|id| !local.contains_key(id) && planned.insert(id.clone())) .collect(); if new_ids.is_empty() { continue; diff --git a/tests/mock_exchange_graph.rs b/tests/mock_exchange_graph.rs index 7e03442..044813a 100644 --- a/tests/mock_exchange_graph.rs +++ b/tests/mock_exchange_graph.rs @@ -713,6 +713,105 @@ fn integration_full_run_mail_only_imports_mime_via_value() { assert_eq!(emails, 1, "expected one email created, got {emails}"); } +#[test] +fn integration_duplicate_message_id_does_not_abort_run() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid-1","userPrincipalName":"alice@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let _well_known: Vec = [ + "inbox", + "drafts", + "sentitems", + "deleteditems", + "junkemail", + "archive", + ] + .iter() + .map(|name| { + let path = format!("/me/mailFolders/{name}?$select=id"); + server + .mock("GET", path.as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(if *name == "inbox" { + r#"{"id":"FMAIL"}"# + } else { + r#"{"id":"OTHER"}"# + }) + .create() + }) + .collect(); + let _ids = server + .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"MSG-1"},{"id":"MSG-1"}]}"#) + .create(); + let mime = "From: a@x\r\nTo: b@x\r\nSubject: hi\r\nDate: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: \r\n\r\nhello"; + let _value = server + .mock("GET", "/me/messages/MSG-1/$value") + .match_header("accept", "text/plain") + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(mime) + .create(); + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let archive_path = tmp.path().to_owned(); + let common = vandelay::sync::CommonConfig { + archive: archive_path.clone(), + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "alice@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + drop(tmp); + let summary = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let emails = summary + .per_type + .iter() + .find(|(t, _)| *t == "email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(emails, 1, "expected one email created, got {emails}"); +} + #[test] fn integration_full_run_is_convergent_on_second_invocation() { let mut server = Server::new();