diff --git a/CHANGELOG.md b/CHANGELOG.md index 0709ff4..8cae36e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,39 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a complete; the checksum shows two archives -- an export and a re-import, say -- hold the very same mail. It exits 8 when the archive is not sound. +### Changed +- Upstream v1.0.16 is merged in. +- Outlook categories become valid JMAP keywords, the same way IMAP + `--mime-keywords` does it (`Catégorie Rouge` is `cat_gorie_rouge`), and + duplicates are dropped. Before, only the case was lowered, so `Project + review` gave a keyword JMAP doesn't allow. A read-receipt request no longer + adds `$notified`. This is for EWS and Graph. + +### Fixed +- Exchange EWS and Graph: all-day events and birthdays landed a day early + east of UTC. The date is now taken in the event's time zone. Outlook's + "no date" (year 4500 or later) is no longer imported as a real date. + Archives imported before this keep the wrong dates until the item changes + at the source, so re-import into a fresh archive to correct them. +- Exchange EWS: Exchange 2019 servers were asked for a schema version that + doesn't exist, so every call was refused once and sent again. +- Exchange EWS: hidden system folders (`Recipient Cache`, `GAL Contacts`) + were imported as mailboxes and address books. They are skipped now. +- Exchange EWS: a mail folder under a calendar or contacts folder could stop + a later run with a FOREIGN KEY error. It is filed under its nearest mail + folder on every run. +- Exchange EWS: an attendee's or contact's ItemId could replace the item's + own, giving `UNIQUE constraint failed`. Tasks and posts in a listing shifted + the ids around them. One item that failed to parse or store stopped the rest + of its folder; it is now logged and counted, and the folder goes on. +- Exchange EWS: event attachments, your own response to a meeting and an + event's private flag were never fetched. They are imported now. +- Exchange EWS: a folder with no class (Outlook's `(This computer only)` + contacts) was always taken for mail, and contacts, appointments and tasks + in a mail folder such as Deleted Items were stored as email. A class-less + folder now takes its kind from its items, and mail folders skip non-mail + items, also in `--estimate`. + ## [27.1-rc.13] The thirteenth release candidate of the 27.1 series: shell completions and a diff --git a/src/exchange/date.rs b/src/exchange/date.rs index d0a9812..077c4ba 100644 --- a/src/exchange/date.rs +++ b/src/exchange/date.rs @@ -4,7 +4,52 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta, TimeZone}; +use chrono_tz::Tz; + const END_OF_DAY: &str = "23:59:59"; +const HALF_DAY: TimeDelta = TimeDelta::hours(12); +const OUTLOOK_NO_DATE_YEAR: i32 = 4500; + +pub fn nearest_midnight(value: &str, iana: Option<&str>) -> Option { + let trimmed = value.trim(); + let local = match DateTime::parse_from_rfc3339(trimmed) { + Ok(instant) => match iana.and_then(|tz| tz.parse::().ok()) { + Some(tz) => instant.with_timezone(&tz).naive_local(), + None => instant.naive_local(), + }, + Err(_) => NaiveDateTime::parse_from_str(trimmed, "%Y-%m-%dT%H:%M:%S%.f") + .or_else(|_| { + NaiveDate::parse_from_str(trimmed, "%Y-%m-%d").map(|d| d.and_time(NaiveTime::MIN)) + }) + .ok()?, + }; + local.checked_add_signed(HALF_DAY).map(|dt| dt.date()) +} + +pub fn all_day_date(utc: &str, iana: Option<&str>) -> Option { + let trimmed = utc.trim(); + let instant = DateTime::parse_from_rfc3339(trimmed) + .map(|dt| dt.naive_utc()) + .or_else(|_| NaiveDateTime::parse_from_str(trimmed, "%Y-%m-%dT%H:%M:%S%.f")) + .ok()?; + if instant.time() == NaiveTime::MIN { + return Some(instant.date()); + } + let local = match iana.and_then(|tz| tz.parse::().ok()) { + Some(tz) => tz.from_utc_datetime(&instant).naive_local(), + None => instant, + }; + local.checked_add_signed(HALF_DAY).map(|dt| dt.date()) +} + +pub fn is_outlook_no_date(value: &str) -> bool { + value + .trim() + .split_once('-') + .and_then(|(year, _)| year.parse::().ok()) + .is_some_and(|year| year >= OUTLOOK_NO_DATE_YEAR) +} pub fn recurrence_until(raw: &str) -> Option { let trimmed = raw.trim(); @@ -57,6 +102,105 @@ fn two_digits(value: &str) -> Option { mod tests { use super::*; + fn date(y: i32, m: u32, d: u32) -> Option { + NaiveDate::from_ymd_opt(y, m, d) + } + + #[test] + fn utc_instant_rounds_to_local_midnight_east_of_utc() { + assert_eq!( + nearest_midnight("2026-11-11T23:00:00Z", Some("Europe/Paris")), + date(2026, 11, 12) + ); + assert_eq!( + nearest_midnight("2026-07-11T22:00:00Z", Some("Europe/Paris")), + date(2026, 7, 12) + ); + } + + #[test] + fn utc_instant_rounds_to_local_midnight_west_of_utc() { + assert_eq!( + nearest_midnight("2026-11-12T05:00:00Z", Some("America/New_York")), + date(2026, 11, 12) + ); + } + + #[test] + fn zone_off_by_an_hour_still_gives_the_same_day() { + assert_eq!( + nearest_midnight("2026-11-11T23:00:00Z", Some("Europe/London")), + date(2026, 11, 12) + ); + assert_eq!( + nearest_midnight("2026-11-11T23:00:00Z", Some("Europe/Helsinki")), + date(2026, 11, 12) + ); + } + + #[test] + fn utc_instant_without_zone_rounds_in_utc() { + assert_eq!( + nearest_midnight("1990-04-14T22:00:00Z", None), + date(1990, 4, 15) + ); + assert_eq!( + nearest_midnight("1990-04-15T05:00:00Z", None), + date(1990, 4, 15) + ); + } + + #[test] + fn floating_and_bare_dates_are_kept() { + assert_eq!( + nearest_midnight("2025-12-25T00:00:00", Some("Europe/Paris")), + date(2025, 12, 25) + ); + assert_eq!(nearest_midnight("2025-12-25", None), date(2025, 12, 25)); + assert_eq!(nearest_midnight("garbage", None), None); + } + + #[test] + fn all_day_midnight_is_taken_as_the_date() { + assert_eq!( + all_day_date("2026-11-12T00:00:00.0000000", Some("Pacific/Auckland")), + date(2026, 11, 12) + ); + assert_eq!( + all_day_date("2026-11-12T00:00:00Z", Some("America/New_York")), + date(2026, 11, 12) + ); + } + + #[test] + fn all_day_utc_instant_moves_to_the_local_date() { + assert_eq!( + all_day_date("2026-11-11T23:00:00.0000000", Some("Europe/Paris")), + date(2026, 11, 12) + ); + assert_eq!( + all_day_date("2026-11-11T11:00:00Z", Some("Pacific/Auckland")), + date(2026, 11, 12) + ); + assert_eq!( + all_day_date("2026-11-12T05:00:00.0000000", Some("America/New_York")), + date(2026, 11, 12) + ); + assert_eq!( + all_day_date("2026-11-11T23:00:00", None), + date(2026, 11, 12) + ); + assert_eq!(all_day_date("garbage", None), None); + } + + #[test] + fn outlook_no_date_sentinel() { + assert!(is_outlook_no_date("4500-08-31T23:00:00Z")); + assert!(is_outlook_no_date("4501-01-01T00:00:00Z")); + assert!(!is_outlook_no_date("2026-11-12T00:00:00Z")); + assert!(!is_outlook_no_date("garbage")); + } + #[test] fn bare_date_gets_end_of_day() { assert_eq!( diff --git a/src/exchange_ews/calendar_map.rs b/src/exchange_ews/calendar_map.rs index ac7b973..99e49c9 100644 --- a/src/exchange_ews/calendar_map.rs +++ b/src/exchange_ews/calendar_map.rs @@ -5,9 +5,11 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use chrono::NaiveDate; use serde_json::{Map, Value, json}; use crate::exchange::address::{as_smtp_address, extract_smtp_address}; +use crate::exchange::date::nearest_midnight; use crate::exchange::jscalendar::{ drop_calendar_address_dependents, is_override_ignored, synthetic_attendee_address, }; @@ -22,20 +24,24 @@ pub struct EventValue { } pub fn to_jscalendar(raw: &CalendarItemRaw) -> EventValue { - to_jscalendar_with_exceptions(raw, &[]) + to_jscalendar_with_exceptions(raw, &[], None) } pub fn to_jscalendar_with_exceptions( raw: &CalendarItemRaw, modified_full: &[CalendarItemRaw], + owner_smtp: Option<&str>, ) -> EventValue { let iana = raw .start_tz .as_deref() .map(|tz| resolve_to_iana(tz).unwrap_or_else(|| "Etc/UTC".to_owned())); - let mut event = build_event_map(raw, iana.as_deref()); - if let Some(overrides) = build_recurrence_overrides(&event, raw, modified_full, iana.as_deref()) - { + let ctx = EventCtx { + iana: iana.as_deref(), + owner_smtp, + }; + let mut event = build_event_map(raw, ctx); + if let Some(overrides) = build_recurrence_overrides(&event, raw, modified_full, ctx) { event.insert("recurrenceOverrides".to_owned(), overrides); } EventValue { @@ -45,7 +51,14 @@ pub fn to_jscalendar_with_exceptions( } } -fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map { +#[derive(Clone, Copy)] +struct EventCtx<'a> { + iana: Option<&'a str>, + owner_smtp: Option<&'a str>, +} + +fn build_event_map(raw: &CalendarItemRaw, ctx: EventCtx<'_>) -> Map { + let iana = ctx.iana; let mut event = Map::new(); event.insert("@type".to_owned(), Value::String("Event".to_owned())); if let Some(uid) = raw.uid.as_ref() { @@ -100,18 +113,12 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map all_day_span_days(s, e).max(1), - _ => 1, - }; + let days = all_day_span_days(start, raw.end.as_deref(), iana); event.insert("duration".to_owned(), Value::String(format!("P{days}D"))); } else { if let Some(start) = raw.start.as_ref() { @@ -129,6 +136,12 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map "free", @@ -185,6 +198,25 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map "accepted", - "Tentative" => "tentative", - "Decline" => "declined", - "Organizer" => "accepted", - "NoResponseReceived" => "needs-action", - _ => "needs-action", - }; + if let Some(rt) = att.response_type.as_deref() { + let mapped = participation_status(rt).unwrap_or("needs-action"); p.insert( "participationStatus".to_owned(), Value::String(mapped.to_owned()), @@ -326,6 +351,35 @@ fn add_attendees( } } +fn participation_status(response_type: &str) -> Option<&'static str> { + hashify::fnc_map!( + response_type.as_bytes(), + "Accept" => { return Some("accepted"); }, + "Tentative" => { return Some("tentative"); }, + "Decline" => { return Some("declined"); }, + "Organizer" => { return Some("accepted"); }, + _ => {} + ); + None +} + +fn own_response_status(my_response_type: &str) -> Option<&'static str> { + if my_response_type == "Organizer" { + return None; + } + participation_status(my_response_type) +} + +fn privacy_for(sensitivity: &str) -> &'static str { + hashify::fnc_map!( + sensitivity.as_bytes(), + "Private" => { return "private"; }, + "Confidential" => { return "secret"; }, + _ => {} + ); + "public" +} + const NOT_PATCHED_PER_OCCURRENCE: &[&str] = &["created", "updated"]; fn is_override_excluded(key: &str) -> bool { @@ -336,8 +390,9 @@ fn build_recurrence_overrides( base_event: &Map, raw: &CalendarItemRaw, modified_full: &[CalendarItemRaw], - iana: Option<&str>, + ctx: EventCtx<'_>, ) -> Option { + let iana = ctx.iana; if raw.modified_occurrences.is_empty() && raw.deleted_occurrences.is_empty() { return None; } @@ -347,7 +402,7 @@ fn build_recurrence_overrides( .original_start .as_deref() .or(occ.start.as_deref()) - .map(|s| to_local_datetime_in(s, iana)) + .map(|s| occurrence_key(raw, s, iana)) else { continue; }; @@ -356,15 +411,15 @@ fn build_recurrence_overrides( .find(|f| !occ.item_id.id.is_empty() && f.id.id == occ.item_id.id); let patch = match full { Some(full) => { - let occ_event = build_event_map(full, iana); + let occ_event = build_event_map(full, ctx); override_patch(&occ_event, base_event, &key) } - None => time_only_patch(occ, iana), + None => time_only_patch(raw, occ, iana), }; map.insert(key, Value::Object(patch)); } for occ in &raw.deleted_occurrences { - let Some(key) = occ.start.as_deref().map(|s| to_local_datetime_in(s, iana)) else { + let Some(key) = occ.start.as_deref().map(|s| occurrence_key(raw, s, iana)) else { continue; }; map.insert(key, json!({"excluded": true})); @@ -407,8 +462,23 @@ fn override_patch( patch } -fn time_only_patch(occ: &RawOccurrence, iana: Option<&str>) -> Map { +fn time_only_patch( + series: &CalendarItemRaw, + occ: &RawOccurrence, + iana: Option<&str>, +) -> Map { let mut o = Map::new(); + if series.is_all_day_event == Some(true) { + let start = occ.start.as_deref().and_then(|s| nearest_midnight(s, iana)); + if let Some(start) = start { + o.insert("start".to_owned(), Value::String(all_day_key(start))); + } + if occ.end.is_some() { + let days = all_day_span_days(start, occ.end.as_deref(), iana); + o.insert("duration".to_owned(), Value::String(format!("P{days}D"))); + } + return o; + } if let Some(s) = occ.start.as_ref() { o.insert( "start".to_owned(), @@ -423,6 +493,19 @@ fn time_only_patch(occ: &RawOccurrence, iana: Option<&str>) -> Map) -> String { + if series.is_all_day_event == Some(true) + && let Some(date) = nearest_midnight(start, iana) + { + return all_day_key(date); + } + to_local_datetime_in(start, iana) +} + +fn all_day_key(date: NaiveDate) -> String { + format!("{date}T00:00:00") +} + fn normalise_utc_datetime(s: &str) -> String { let trimmed = s.trim(); let stripped = trimmed @@ -487,32 +570,11 @@ fn convert_utc_to_local(s: &str, iana: &str) -> Option { Some(local.format("%Y-%m-%dT%H:%M:%S").to_string()) } -fn all_day_span_days(start: &str, end: &str) -> u32 { - fn date_only(s: &str) -> &str { - s.split('T').next().unwrap_or(s) +fn all_day_span_days(start: Option, end: Option<&str>, iana: Option<&str>) -> i64 { + match (start, end.and_then(|e| nearest_midnight(e, iana))) { + (Some(start), Some(end)) => (end - start).num_days().max(1), + _ => 1, } - fn ymd(s: &str) -> Option<(i64, u32, u32)> { - let mut p = s.split('-'); - let y: i64 = p.next()?.parse().ok()?; - let m: u32 = p.next()?.parse().ok()?; - let d: u32 = p.next()?.parse().ok()?; - Some((y, m, d)) - } - fn to_days(y: i64, m: u32, d: u32) -> i64 { - let (mut y, m) = if m <= 2 { - (y - 1, m as i64 + 12) - } else { - (y, m as i64) - }; - y += 4800; - let mm = m + 1; - 365 * y + y / 4 - y / 100 + y / 400 + 30 * mm + 3 * (mm + 1) / 5 + d as i64 - 32045 - } - let (Some(s), Some(e)) = (ymd(date_only(start)), ymd(date_only(end))) else { - return 1; - }; - let diff = to_days(e.0, e.1, e.2) - to_days(s.0, s.1, s.2); - if diff <= 0 { 1 } else { diff as u32 } } fn duration_iso8601(start: &str, end: &str) -> Option { @@ -844,7 +906,7 @@ mod tests { end: Some("2025-06-18T14:30:00Z".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar_with_exceptions(&master, std::slice::from_ref(&full_occ)).data; + let v = to_jscalendar_with_exceptions(&master, std::slice::from_ref(&full_occ), None).data; let ov = &v["recurrenceOverrides"]["2025-06-18T14:00:00"]; assert_eq!( ov["title"], "Sprint Retro", @@ -1108,6 +1170,80 @@ mod tests { assert_eq!(bob["roles"]["required"], true); } + fn invited_raw(my_response: &str) -> CalendarItemRaw { + CalendarItemRaw { + uid: Some("uid-mine".to_owned()), + start: Some("2025-06-15T14:00:00Z".to_owned()), + end: Some("2025-06-15T15:00:00Z".to_owned()), + organizer_smtp: Some("alice@example.com".to_owned()), + my_response_type: Some(my_response.to_owned()), + required_attendees: vec![ + crate::exchange_ews::parse::RawAttendee { + email: Some("Me@Example.com".to_owned()), + routing_type: None, + name: None, + response_type: Some("Unknown".to_owned()), + }, + crate::exchange_ews::parse::RawAttendee { + email: Some("bob@example.com".to_owned()), + routing_type: None, + name: None, + response_type: Some("Unknown".to_owned()), + }, + ], + ..CalendarItemRaw::default() + } + } + + fn status_of(v: &Value, email: &str) -> Value { + v["participants"] + .as_object() + .unwrap() + .values() + .find(|p| p["email"] == email) + .unwrap()["participationStatus"] + .clone() + } + + #[test] + fn own_response_sets_the_mailbox_owner_participation_status() { + let raw = invited_raw("Accept"); + let v = to_jscalendar_with_exceptions(&raw, &[], Some("me@example.com")).data; + assert_eq!(status_of(&v, "Me@Example.com"), "accepted"); + assert_eq!(status_of(&v, "bob@example.com"), "needs-action"); + let v = to_jscalendar(&raw).data; + assert_eq!(status_of(&v, "Me@Example.com"), "needs-action"); + let raw = invited_raw("NoResponseReceived"); + let v = to_jscalendar_with_exceptions(&raw, &[], Some("me@example.com")).data; + assert_eq!(status_of(&v, "Me@Example.com"), "needs-action"); + } + + #[test] + fn sensitivity_maps_to_privacy() { + for (sensitivity, privacy) in [ + ("Private", "private"), + ("Confidential", "secret"), + ("Personal", "public"), + ("Normal", "public"), + ] { + let raw = CalendarItemRaw { + uid: Some("uid-p".to_owned()), + sensitivity: Some(sensitivity.to_owned()), + ..CalendarItemRaw::default() + }; + assert_eq!( + to_jscalendar(&raw).data["privacy"], + privacy, + "{sensitivity}" + ); + } + let raw = CalendarItemRaw { + uid: Some("uid-p".to_owned()), + ..CalendarItemRaw::default() + }; + assert!(to_jscalendar(&raw).data.get("privacy").is_none()); + } + #[test] fn attendees_lose_calendar_addresses_when_no_organizer_is_known() { let raw = CalendarItemRaw { @@ -1140,9 +1276,65 @@ mod tests { #[test] fn all_day_span_helper_is_inclusive_of_start() { - assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1); - assert_eq!(all_day_span_days("2025-07-04", "2025-07-05"), 1); - assert_eq!(all_day_span_days("2025-07-04", "2025-07-07"), 3); - assert_eq!(all_day_span_days("bad", "input"), 1); + let start = NaiveDate::from_ymd_opt(2025, 7, 4); + assert_eq!(all_day_span_days(start, Some("2025-07-04"), None), 1); + assert_eq!(all_day_span_days(start, Some("2025-07-05"), None), 1); + assert_eq!(all_day_span_days(start, Some("2025-07-07"), None), 3); + assert_eq!(all_day_span_days(None, Some("2025-07-07"), None), 1); + assert_eq!(all_day_span_days(start, Some("bad"), None), 1); + } + + #[test] + fn all_day_event_east_of_utc_keeps_its_local_date() { + let raw = CalendarItemRaw { + uid: Some("uid-paris".to_owned()), + is_all_day_event: Some(true), + start: Some("2026-11-11T23:00:00Z".to_owned()), + end: Some("2026-11-12T23:00:00Z".to_owned()), + start_tz: Some("Romance Standard Time".to_owned()), + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["start"], "2026-11-12T00:00:00"); + assert_eq!(v["duration"], "P1D"); + assert!(v.get("timeZone").is_none()); + } + + #[test] + fn all_day_series_exceptions_match_generated_occurrences() { + let raw = CalendarItemRaw { + uid: Some("uid-paris-series".to_owned()), + is_all_day_event: Some(true), + start: Some("2026-11-11T23:00:00Z".to_owned()), + end: Some("2026-11-12T23:00:00Z".to_owned()), + start_tz: Some("Romance Standard Time".to_owned()), + recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + pattern: Some(RecurrencePattern::Daily { interval: 1 }), + range: Some(RecurrenceRange::Numbered { + start_date: "2026-11-12".to_owned(), + number_of_occurrences: 5, + }), + }), + modified_occurrences: vec![RawOccurrence { + item_id: Default::default(), + start: Some("2026-11-14T23:00:00Z".to_owned()), + end: Some("2026-11-15T23:00:00Z".to_owned()), + original_start: Some("2026-11-12T23:00:00Z".to_owned()), + }], + deleted_occurrences: vec![RawOccurrence { + item_id: Default::default(), + start: Some("2026-11-13T23:00:00Z".to_owned()), + end: None, + original_start: None, + }], + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["start"], "2026-11-12T00:00:00"); + let over = v["recurrenceOverrides"].as_object().unwrap(); + assert_eq!(over.len(), 2); + assert_eq!(over["2026-11-13T00:00:00"]["start"], "2026-11-15T00:00:00"); + assert_eq!(over["2026-11-13T00:00:00"]["duration"], "P1D"); + assert_eq!(over["2026-11-14T00:00:00"]["excluded"], true); } } diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs index 01f83e9..f62ac5a 100644 --- a/src/exchange_ews/client.rs +++ b/src/exchange_ews/client.rs @@ -37,7 +37,7 @@ struct Inner { retry: RetryPolicy, rate_limit: RateLimitState, pace: Pacer, - version: Mutex, + version: Mutex, log_level: AtomicU8, retries_total: AtomicU64, retry_after_sleeps: AtomicU64, @@ -56,6 +56,12 @@ impl Inner { } } +#[derive(Debug, Clone, Copy)] +struct VersionState { + current: ServerVersion, + accepted_ceiling: Option, +} + #[derive(Clone)] pub struct EwsClient { inner: Arc, @@ -92,7 +98,10 @@ impl EwsClient { retry, rate_limit: RateLimitState::new(), pace: Pacer::new(), - version: Mutex::new(ServerVersion::Exchange2013Sp1), + version: Mutex::new(VersionState { + current: ServerVersion::Exchange2013Sp1, + accepted_ceiling: None, + }), log_level: AtomicU8::new(LEVEL_DEFAULT), retries_total: AtomicU64::new(0), retry_after_sleeps: AtomicU64::new(0), @@ -138,7 +147,14 @@ impl EwsClient { pub fn set_server_version(&self, version: ServerVersion) { if let Ok(mut g) = self.inner.version.lock() { - *g = version; + g.current = g.accepted_ceiling.map_or(version, |c| version.min(c)); + } + } + + fn pin_accepted_version(&self, version: ServerVersion) { + if let Ok(mut g) = self.inner.version.lock() { + g.current = version; + g.accepted_ceiling = Some(version); } } @@ -146,7 +162,7 @@ impl EwsClient { self.inner .version .lock() - .map(|g| *g) + .map(|g| g.current) .unwrap_or(ServerVersion::Exchange2013Sp1) } @@ -215,6 +231,7 @@ impl EwsClient { let logger = self.logger(); let policy = self.inner.retry; let mut attempt: u32 = 0; + let mut downgraded = false; loop { self.inner.rate_limit.cooldown().wait(); self.inner.pace.wait(); @@ -232,6 +249,9 @@ impl EwsClient { self.inner.rate_limit.on_success(); match summary { Ok(EnvelopeKind::Body { version }) => { + if downgraded { + self.pin_accepted_version(sent_version); + } if let Some(v) = version.to_server_version() { self.set_server_version(v); } @@ -256,6 +276,7 @@ impl EwsClient { } FaultStep::Downgrade(v) => { self.set_server_version(v); + downgraded = true; continue; } } @@ -282,6 +303,7 @@ impl EwsClient { } FaultStep::Downgrade(v) => { self.set_server_version(v); + downgraded = true; continue; } } @@ -595,8 +617,22 @@ mod tests { RetryPolicy::new(0), CertOverride::none(), ); - c.set_server_version(ServerVersion::Exchange2019); - assert_eq!(c.server_version(), ServerVersion::Exchange2019); + c.set_server_version(ServerVersion::Exchange2016); + assert_eq!(c.server_version(), ServerVersion::Exchange2016); + } + + #[test] + fn accepted_downgrade_caps_reported_version() { + let c = EwsClient::new( + Auth::Bearer { token: "t".into() }, + RetryPolicy::new(0), + CertOverride::none(), + ); + c.pin_accepted_version(ServerVersion::Exchange2013Sp1); + c.set_server_version(ServerVersion::Exchange2016); + assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); + c.set_server_version(ServerVersion::Exchange2010Sp2); + assert_eq!(c.server_version(), ServerVersion::Exchange2010Sp2); } #[test] diff --git a/src/exchange_ews/contact_map.rs b/src/exchange_ews/contact_map.rs index 4747efc..07db037 100644 --- a/src/exchange_ews/contact_map.rs +++ b/src/exchange_ews/contact_map.rs @@ -5,8 +5,10 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use chrono::Datelike; use serde_json::{Map, Value, json}; +use crate::exchange::date::{is_outlook_no_date, nearest_midnight}; use crate::exchange_ews::parse::{ContactItemRaw, RawContactAddress}; pub fn synthetic_uid(item_id: &str) -> String { @@ -316,18 +318,17 @@ fn im_service_from_key(key: &str) -> Option { } fn partial_date(s: &str) -> Option { - let head = s.split('T').next().unwrap_or(s); - let mut parts = head.splitn(3, '-'); - let year: u32 = parts.next()?.parse().ok()?; - let month: u32 = parts.next()?.parse().ok()?; - let day: u32 = parts.next()?.parse().ok()?; + if is_outlook_no_date(s) { + return None; + } + let date = nearest_midnight(s, None)?; let mut o = Map::new(); o.insert("@type".to_owned(), Value::String("PartialDate".to_owned())); - if year > 0 { - o.insert("year".to_owned(), Value::from(year)); + if date.year() > 0 { + o.insert("year".to_owned(), Value::from(date.year())); } - o.insert("month".to_owned(), Value::from(month)); - o.insert("day".to_owned(), Value::from(day)); + o.insert("month".to_owned(), Value::from(date.month())); + o.insert("day".to_owned(), Value::from(date.day())); Some(Value::Object(o)) } @@ -407,6 +408,47 @@ mod tests { assert_eq!(v["anniversaries"]["wedding"]["kind"], "wedding"); } + #[test] + fn birthday_stored_as_local_midnight_in_utc_keeps_its_day() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("1990-04-14T22:00:00Z".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!( + v["anniversaries"]["birth"]["date"], + json!({"@type": "PartialDate", "year": 1990, "month": 4, "day": 15}) + ); + } + + #[test] + fn year_zero_birthday_has_no_year() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("0000-04-15T00:00:00Z".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!( + v["anniversaries"]["birth"]["date"], + json!({"@type": "PartialDate", "month": 4, "day": 15}) + ); + } + + #[test] + fn outlook_no_date_anniversary_is_dropped() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("1990-04-15T00:00:00Z".to_owned()), + wedding_anniversary: Some("4501-01-01T00:00:00Z".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert!(v["anniversaries"].get("wedding").is_none()); + assert_eq!(v["anniversaries"]["birth"]["date"]["day"], 15); + } + #[test] fn office_location_is_added_as_org_unit() { let raw = ContactItemRaw { diff --git a/src/exchange_ews/parse.rs b/src/exchange_ews/parse.rs index 6267033..beb0679 100644 --- a/src/exchange_ews/parse.rs +++ b/src/exchange_ews/parse.rs @@ -263,6 +263,7 @@ pub struct FolderEntry { pub folder_class: String, pub total_count: Option, pub child_count: Option, + pub is_hidden: bool, } impl Default for FolderEntry { @@ -278,6 +279,7 @@ impl Default for FolderEntry { folder_class: String::new(), total_count: None, child_count: None, + is_hidden: false, } } } @@ -409,6 +411,7 @@ fn parse_folder_element( let mut buf = Vec::new(); let mut depth: u32 = 1; let mut current: Option<&'static str> = None; + let mut in_hidden_ext = false; let mut cur = String::new(); loop { buf.clear(); @@ -447,6 +450,14 @@ fn parse_folder_element( } else if local.eq_ignore_ascii_case("ChildFolderCount") { current = Some("childCount"); cur.clear(); + } else if local.eq_ignore_ascii_case("ExtendedFieldURI") { + in_hidden_ext = attr_value(e, "PropertyTag").is_some_and(|tag| { + let tag = tag.trim(); + tag.eq_ignore_ascii_case("0x10f4") || tag == "4340" + }); + } else if in_hidden_ext && local.eq_ignore_ascii_case("Value") { + current = Some("hidden"); + cur.clear(); } else { current = None; } @@ -463,6 +474,7 @@ fn parse_folder_element( "folderClass" => entry.folder_class = text.trim().to_owned(), "totalCount" => entry.total_count = text.trim().parse().ok(), "childCount" => entry.child_count = text.trim().parse().ok(), + "hidden" => entry.is_hidden = matches!(text.trim(), "true" | "1"), _ => {} } } @@ -505,6 +517,7 @@ pub struct ItemEntry { pub id: ItemId, /// `item:Size` in bytes, when the server returned it. pub size: Option, + pub item_class: Option, } pub fn parse_find_item_response(body: &[u8]) -> Result { @@ -513,6 +526,9 @@ pub fn parse_find_item_response(body: &[u8]) -> Result = None; + let mut reading_class = false; let mut reading_size = false; loop { buf.clear(); @@ -520,6 +536,7 @@ pub fn parse_find_item_response(body: &[u8]) -> Result { + let level = depth.open(matches!(ev, Event::Empty(_))); let local = e.local_name().as_ref().to_owned(); if is(ns_kind, &local, Ns::Messages, "RootFolder") { in_root = true; @@ -530,24 +547,35 @@ pub fn parse_find_item_response(body: &[u8]) -> Result { + if let Some(last) = out.items.last_mut() { + let text: &str = t; + last.item_class = Some(text.trim().to_owned()); + } + } Event::Text(ref t) if reading_size => { if let Some(last) = out.items.last_mut() { let text: &str = t; @@ -555,11 +583,13 @@ pub fn parse_find_item_response(body: &[u8]) -> Result { + depth.close(); + reading_class = false; + reading_size = false; let local = e.local_name().as_ref().to_owned(); if local.eq_ignore_ascii_case("RootFolder") { in_root = false; - } else if local.eq_ignore_ascii_case("Size") { - reading_size = false; + items_level = None; } } Event::Eof => break, @@ -584,20 +614,51 @@ pub(crate) fn entity_to_char(g: &BytesRef) -> Option { } fn is_item_element(local: &str) -> bool { - matches!( - local.to_ascii_lowercase().as_str(), - "message" - | "calendaritem" - | "contact" - | "distributionlist" - | "meetingrequest" - | "meetingresponse" - | "meetingmessage" - | "meetingcancellation" - | "item" + hashify::set_ignore_case!( + local.as_bytes(), + "Item", + "Message", + "SharingMessage", + "CalendarItem", + "Contact", + "DistributionList", + "MeetingMessage", + "MeetingRequest", + "MeetingResponse", + "MeetingCancellation", + "Task", + "PostItem", ) } +#[derive(Default)] +struct ItemDepth { + depth: usize, + item_level: Option, +} + +impl ItemDepth { + fn open(&mut self, is_empty: bool) -> usize { + let level = self.depth + 1; + if !is_empty { + self.depth = level; + } + level + } + + fn close(&mut self) { + self.depth = self.depth.saturating_sub(1); + } + + fn enter_item(&mut self, level: usize) { + self.item_level = Some(level); + } + + fn is_item_child(&self, level: usize) -> bool { + self.item_level.is_some_and(|item| level == item + 1) + } +} + #[derive(Debug, Clone)] pub struct ResponseMessage { pub success: bool, @@ -816,10 +877,23 @@ pub struct SyncFolderItemsResponse { #[derive(Debug, Clone)] pub enum SyncChange { - Create { id: ItemId, element: String }, - Update { id: ItemId, element: String }, - Delete { id: ItemId }, - ReadFlagChange { id: ItemId, is_read: bool }, + Create { + id: ItemId, + element: String, + item_class: Option, + }, + Update { + id: ItemId, + element: String, + item_class: Option, + }, + Delete { + id: ItemId, + }, + ReadFlagChange { + id: ItemId, + is_read: bool, + }, } pub fn parse_sync_folder_items_response(body: &[u8]) -> Result { @@ -834,6 +908,7 @@ pub fn parse_sync_folder_items_response(body: &[u8]) -> Result = None; let mut reading_is_read = false; let mut pending_is_read: Option = None; + let mut reading_item_class = false; loop { buf.clear(); let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; @@ -853,11 +928,13 @@ pub fn parse_sync_folder_items_response(body: &[u8]) -> Result Result Result reading_is_read = false, + "itemclass" => reading_item_class = false, "changes" => in_changes = false, _ => {} } @@ -915,6 +995,14 @@ pub fn parse_sync_folder_items_response(body: &[u8]) -> Result sync_state = text, Some("includesLast") => more = !matches!(text.trim(), "true" | "1"), @@ -958,7 +1046,6 @@ pub struct MessageItem { pub date_time_received: Option, pub is_read: Option, pub is_draft: Option, - pub is_read_receipt_requested: Option, pub categories: Vec, pub flag_status: Option, } @@ -973,6 +1060,7 @@ pub fn parse_message_item(inner_xml: &str) -> Result { let mut category_collecting = false; let mut in_flag = false; let mut in_flagstatus_ext = false; + let mut depth = ItemDepth::default(); let mut cur = String::new(); loop { buf.clear(); @@ -982,12 +1070,16 @@ pub fn parse_message_item(inner_xml: &str) -> Result { Event::Start(ref e) | Event::Empty(ref e) => { let local = e.local_name().as_ref().to_owned(); let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); if item.element.is_empty() && is_item_element(&local) { item.element = local.clone(); + depth.enter_item(level); } else if ns_kind == Ns::Types { cur.clear(); if local.eq_ignore_ascii_case("ItemId") { - capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + if depth.is_item_child(level) { + capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + } } else if local.eq_ignore_ascii_case("ParentFolderId") { let mut pid = String::new(); let mut ck = String::new(); @@ -1006,8 +1098,6 @@ pub fn parse_message_item(inner_xml: &str) -> Result { current = Some("isRead"); } else if local.eq_ignore_ascii_case("IsDraft") { current = Some("isDraft"); - } else if local.eq_ignore_ascii_case("IsReadReceiptRequested") { - current = Some("readReceipt"); } else if local.eq_ignore_ascii_case("Categories") { category_collecting = true; } else if category_collecting && local.eq_ignore_ascii_case("String") { @@ -1032,6 +1122,7 @@ pub fn parse_message_item(inner_xml: &str) -> Result { } } Event::End(e) => { + depth.close(); let local = e.local_name().as_ref().to_owned(); let lower = local.to_ascii_lowercase(); if let Some(field) = current.take() { @@ -1045,10 +1136,6 @@ pub fn parse_message_item(inner_xml: &str) -> Result { "received" => item.date_time_received = Some(text), "isRead" => item.is_read = Some(matches!(text.trim(), "true" | "1")), "isDraft" => item.is_draft = Some(matches!(text.trim(), "true" | "1")), - "readReceipt" => { - item.is_read_receipt_requested = - Some(matches!(text.trim(), "true" | "1")); - } "category" => item.categories.push(text), "flagStatus" => item.flag_status = Some(text), "flagStatusExt" if text.trim() == "2" => { @@ -1096,6 +1183,8 @@ pub struct CalendarItemRaw { pub original_start: Option, pub is_all_day_event: Option, pub legacy_free_busy_status: Option, + pub my_response_type: Option, + pub sensitivity: Option, pub location: Option, pub calendar_item_type: Option, pub recurrence_id: Option, @@ -1218,6 +1307,7 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result let mut recurrence_text: Option<&'static str> = None; let mut recurrence = RawRecurrence::default(); let mut pending = PendingRecurrence::default(); + let mut depth = ItemDepth::default(); let mut cur = String::new(); loop { @@ -1228,8 +1318,10 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result Event::Start(ref e) | Event::Empty(ref e) => { let local = e.local_name().as_ref().to_owned(); let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); if item.element.is_empty() && is_item_element(&local) { item.element = local.clone(); + depth.enter_item(level); continue; } if ns_kind != Ns::Types { @@ -1290,7 +1382,7 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result if let Some(occ) = deleted_stack.last_mut() { capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); } - } else { + } else if depth.is_item_child(level) { capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); } } else if local.eq_ignore_ascii_case("ParentFolderId") { @@ -1326,6 +1418,10 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result text_target = Some("isAllDay"); } else if local.eq_ignore_ascii_case("LegacyFreeBusyStatus") { text_target = Some("freeBusy"); + } else if local.eq_ignore_ascii_case("MyResponseType") { + text_target = Some("myResponse"); + } else if local.eq_ignore_ascii_case("Sensitivity") { + text_target = Some("sensitivity"); } else if local.eq_ignore_ascii_case("Location") { text_target = Some("location"); } else if local.eq_ignore_ascii_case("CalendarItemType") { @@ -1424,6 +1520,7 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result } } Event::End(e) => { + depth.close(); let local = e.local_name().as_ref().to_owned(); let lower = local.to_ascii_lowercase(); if !recurrence_path.is_empty() { @@ -1448,6 +1545,8 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result item.is_all_day_event = Some(matches!(text.trim(), "true" | "1")); } "freeBusy" => item.legacy_free_busy_status = Some(text), + "myResponse" => item.my_response_type = Some(text.trim().to_owned()), + "sensitivity" => item.sensitivity = Some(text.trim().to_owned()), "location" => item.location = Some(text), "calendarItemType" => { item.calendar_item_type = CalendarItemType::parse(text.trim()); @@ -1724,6 +1823,7 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { let mut in_members = false; let mut member_mailbox = false; let mut current_member: Option = None; + let mut depth = ItemDepth::default(); let mut cur = String::new(); loop { buf.clear(); @@ -1733,11 +1833,13 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { Event::Start(ref e) | Event::Empty(ref e) => { let local = e.local_name().as_ref().to_owned(); let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); if !seen_root_contact && (local.eq_ignore_ascii_case("Contact") || local.eq_ignore_ascii_case("DistributionList")) { seen_root_contact = true; + depth.enter_item(level); if local.eq_ignore_ascii_case("DistributionList") { item.is_group = true; } @@ -1748,7 +1850,9 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { } cur.clear(); if local.eq_ignore_ascii_case("ItemId") { - capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + if depth.is_item_child(level) { + capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + } } else if local.eq_ignore_ascii_case("ParentFolderId") { let mut pid = String::new(); let mut ck = String::new(); @@ -1868,6 +1972,7 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { } } Event::End(e) => { + depth.close(); let local = e.local_name().as_ref().to_owned(); let lower = local.to_ascii_lowercase(); if let Some(at) = address_text { @@ -2236,6 +2341,162 @@ mod tests { assert_eq!(r.folders[0].parent_id.as_deref(), Some("ROOT")); assert_eq!(r.folders[1].element, FolderElement::CalendarFolder); assert!(!r.more); + assert!(!r.folders[0].is_hidden); + } + + #[test] + fn find_folder_reads_pr_attr_hidden() { + let body = format!( + "\ + NoError\ + \ + \ + \ + IPF.ContactRecipient Cache\ + \ + true\ + \ + IPF.NoteInbox\ + \ + false\ + " + ); + let r = parse_find_folder_response(body.as_bytes()).unwrap(); + assert_eq!(r.folders.len(), 2); + assert!(r.folders[0].is_hidden); + assert_eq!(r.folders[0].display_name, "Recipient Cache"); + assert!(!r.folders[1].is_hidden); + assert_eq!(r.folders[1].display_name, "Inbox"); + } + + #[test] + fn find_item_reads_item_class() { + let body = format!( + "\ + NoError\ + \ + IPM.Contact\ + \ + " + ); + let r = parse_find_item_response(body.as_bytes()).unwrap(); + assert_eq!(r.items[0].item_class.as_deref(), Some("IPM.Contact")); + assert_eq!(r.items[1].item_class, None); + } + + #[test] + fn sync_folder_items_reads_item_class() { + let body = format!( + "\ + NoError\ + AAAA\ + true\ + \ + IPM.Task\ + IPM.Note\ + " + ); + let r = parse_sync_folder_items_response(body.as_bytes()).unwrap(); + let classes: Vec> = + r.changes + .iter() + .map(|c| match c { + SyncChange::Create { item_class, .. } + | SyncChange::Update { item_class, .. } => item_class.as_deref(), + _ => None, + }) + .collect(); + assert_eq!(classes, [Some("IPM.Task"), Some("IPM.Note")]); + } + + #[test] + fn find_item_opens_an_entry_for_every_item_element() { + let body = format!( + "\ + NoError\ + \ + \ + \ + \ + \ + " + ); + let r = parse_find_item_response(body.as_bytes()).unwrap(); + let ids: Vec<(&str, &str)> = r + .items + .iter() + .map(|i| (i.element.as_str(), i.id.id.as_str())) + .collect(); + assert_eq!( + ids, + [ + ("Message", "M1"), + ("Task", "T1"), + ("PostItem", "P1"), + ("FutureItem", "F1") + ] + ); + } + + #[test] + fn find_item_ignores_item_ids_nested_in_mailboxes() { + let body = format!( + "\ + NoError\ + \ + \ + bob@example.com\ + \ + " + ); + let r = parse_find_item_response(body.as_bytes()).unwrap(); + assert_eq!(r.items.len(), 1); + assert_eq!(r.items[0].id.id, "E1"); + } + + #[test] + fn calendar_item_keeps_its_own_id_when_attendees_carry_contact_ids() { + let xml = "\ + \ + alice@example.com\ + Contact\ + \ + bob@example.com\ + Contact\ + "; + let item = parse_calendar_item(xml).unwrap(); + assert_eq!(item.id.id, "EVENT-A"); + assert_eq!(item.id.change_key, "CK"); + assert_eq!(item.required_attendees.len(), 1); + } + + #[test] + fn message_item_keeps_its_own_id_when_recipients_carry_contact_ids() { + let xml = "\ + \ + bob@example.com\ + "; + let item = parse_message_item(xml).unwrap(); + assert_eq!(item.id.id, "MSG-A"); + assert_eq!(item.element, "MeetingRequest"); + } + + #[test] + fn distribution_list_keeps_its_own_id_when_members_carry_contact_ids() { + let xml = "\ + \ + Team\ + Bobbob@example.com\ + Contact\ + "; + let item = parse_contact_item(xml).unwrap(); + assert_eq!(item.id.id, "DL-A"); + assert!(item.is_group); + assert_eq!(item.members.len(), 1); + assert_eq!(item.members[0].email.as_deref(), Some("bob@example.com")); } #[test] @@ -2364,7 +2625,7 @@ mod tests { assert!(!r.more); assert_eq!(r.changes.len(), 3); match &r.changes[0] { - SyncChange::Create { id, element } => { + SyncChange::Create { id, element, .. } => { assert_eq!(id.id, "I1"); assert_eq!(element, "Message"); } @@ -2530,6 +2791,31 @@ mod tests { ); } + #[test] + fn calendar_parses_own_response_sensitivity_and_attachments() { + let body = format!( + "\ + \ + Private\ + \ + agenda.pdfapplication/pdf\ + 2020-01-01T00:00:00\ + \ + 2025-06-01T00:00:00Z\ + Accept\ + " + ); + let parsed = parse_calendar_item(&body).unwrap(); + assert_eq!(parsed.sensitivity.as_deref(), Some("Private")); + assert_eq!(parsed.my_response_type.as_deref(), Some("Accept")); + assert_eq!(parsed.attachments.len(), 1); + assert_eq!(parsed.attachments[0].attachment_id, "ATT1"); + assert_eq!( + parsed.last_modified.as_deref(), + Some("2025-06-01T00:00:00Z") + ); + } + #[test] fn calendar_parses_reminder_resources_and_online_meeting() { let body = format!( diff --git a/src/exchange_ews/types.rs b/src/exchange_ews/types.rs index d9dfec5..0eb610a 100644 --- a/src/exchange_ews/types.rs +++ b/src/exchange_ews/types.rs @@ -103,7 +103,6 @@ pub enum ServerVersion { Exchange2013, Exchange2013Sp1, Exchange2016, - Exchange2019, } impl ServerVersion { @@ -116,7 +115,6 @@ impl ServerVersion { ServerVersion::Exchange2013 => "Exchange2013", ServerVersion::Exchange2013Sp1 => "Exchange2013_SP1", ServerVersion::Exchange2016 => "Exchange2016", - ServerVersion::Exchange2019 => "Exchange2019", } } @@ -127,16 +125,12 @@ impl ServerVersion { (14, 1) => ServerVersion::Exchange2010Sp1, (14, 2 | 3) => ServerVersion::Exchange2010Sp2, (15, 0) => ServerVersion::Exchange2013, - (15, 1) => ServerVersion::Exchange2016, - (15, 2) => ServerVersion::Exchange2019, - _ if major >= 15 => ServerVersion::Exchange2019, _ => ServerVersion::Exchange2016, } } pub fn lower(self) -> Option { match self { - ServerVersion::Exchange2019 => Some(ServerVersion::Exchange2016), ServerVersion::Exchange2016 => Some(ServerVersion::Exchange2013Sp1), ServerVersion::Exchange2013Sp1 => Some(ServerVersion::Exchange2013), ServerVersion::Exchange2013 => Some(ServerVersion::Exchange2010Sp2), @@ -148,6 +142,27 @@ impl ServerVersion { } } +const NON_MAIL_ITEM_CLASSES: [&str; 6] = [ + "ipm.contact", + "ipm.distlist", + "ipm.appointment", + "ipm.task", + "ipm.stickynote", + "ipm.activity", +]; + +pub fn is_non_mail_item_class(class: &str) -> bool { + let class = class.trim(); + NON_MAIL_ITEM_CLASSES.iter().any(|prefix| { + class + .get(..prefix.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(prefix)) + && class + .get(prefix.len()..) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('.')) + }) +} + const SKIPPED_CONTAINER_CLASSES: [&str; 8] = [ "ipf.task", "ipf.journal", @@ -190,6 +205,19 @@ impl FolderClass { } } + pub fn from_item_classes<'a>(classes: impl IntoIterator) -> FolderClass { + for class in classes { + let lower = class.trim().to_ascii_lowercase(); + if class_matches(&lower, "ipm.contact") || class_matches(&lower, "ipm.distlist") { + return FolderClass::Contacts; + } + if class_matches(&lower, "ipm.appointment") { + return FolderClass::Calendar; + } + } + FolderClass::Mail + } + pub fn is_mail_fallback(class: &str) -> bool { let lower = class.trim().to_ascii_lowercase(); FolderClass::from_ipf(class) == FolderClass::Mail && !class_matches(&lower, "ipf.note") @@ -309,6 +337,54 @@ impl fmt::Display for ResponseCode { mod tests { use super::*; + #[test] + fn non_mail_item_classes() { + for class in [ + "IPM.Contact", + "IPM.DistList", + "IPM.Appointment", + "IPM.Appointment.Custom", + "IPM.Task", + "ipm.stickynote", + "IPM.Activity", + ] { + assert!(is_non_mail_item_class(class), "{class}"); + } + for class in [ + "IPM.Note", + "IPM.Note.SMIME", + "IPM.Schedule.Meeting.Request", + "IPM.Schedule.Meeting.Canceled", + "IPM.Post", + "REPORT.IPM.Note.NDR", + "IPM.Contactless", + "", + ] { + assert!(!is_non_mail_item_class(class), "{class}"); + } + } + + #[test] + fn folder_kind_from_item_classes() { + assert_eq!( + FolderClass::from_item_classes(["IPM.Note", "IPM.Contact"]), + FolderClass::Contacts + ); + assert_eq!( + FolderClass::from_item_classes(["IPM.DistList"]), + FolderClass::Contacts + ); + assert_eq!( + FolderClass::from_item_classes(["IPM.Appointment"]), + FolderClass::Calendar + ); + assert_eq!( + FolderClass::from_item_classes(["IPM.Note", "IPM.Schedule.Meeting.Request"]), + FolderClass::Mail + ); + assert_eq!(FolderClass::from_item_classes([]), FolderClass::Mail); + } + #[test] fn folder_class_mapping() { assert_eq!(FolderClass::from_ipf("IPF.Note"), FolderClass::Mail); @@ -364,11 +440,15 @@ mod tests { ); assert_eq!( ServerVersion::from_build(15, 2), - ServerVersion::Exchange2019 + ServerVersion::Exchange2016 + ); + assert_eq!( + ServerVersion::from_build(15, 20), + ServerVersion::Exchange2016 ); assert_eq!( ServerVersion::from_build(99, 99), - ServerVersion::Exchange2019 + ServerVersion::Exchange2016 ); } @@ -396,7 +476,7 @@ mod tests { #[test] fn server_version_ladder_descends_to_floor() { - let mut v = ServerVersion::Exchange2019; + let mut v = ServerVersion::Exchange2016; let mut chain = vec![v]; while let Some(next) = v.lower() { chain.push(next); @@ -405,7 +485,6 @@ mod tests { assert_eq!( chain, vec![ - ServerVersion::Exchange2019, ServerVersion::Exchange2016, ServerVersion::Exchange2013Sp1, ServerVersion::Exchange2013, diff --git a/src/exchange_ews/xml.rs b/src/exchange_ews/xml.rs index 979dfa7..bda032f 100644 --- a/src/exchange_ews/xml.rs +++ b/src/exchange_ews/xml.rs @@ -64,6 +64,7 @@ pub fn find_folder_body(parent: FolderRef<'_>, traversal: Traversal) -> String { out.push_str(""); out.push_str(""); out.push_str(""); + out.push_str(""); out.push_str(""); out.push_str(""); write_folder_ref(&mut out, parent); @@ -85,6 +86,9 @@ pub fn get_folder_body(folders: &[FolderRef<'_>], shape: FolderShape) -> String out } +const ITEM_CLASS_PROPERTY: &str = + ""; + pub fn find_item_body( parent: FolderRef<'_>, traversal: Traversal, @@ -94,10 +98,12 @@ pub fn find_item_body( let mut out = String::with_capacity(512); out.push_str("IdOnly\ - \ + \ + \ ", ); out.push_str(""); out.push_str("IdOnly"); + out.push_str(ITEM_CLASS_PROPERTY); out.push_str(""); write_folder_id_element(&mut out, folder); out.push_str(""); @@ -168,7 +175,6 @@ pub fn get_item_body(shape: ItemShape, ids: &[ItemId], version: ServerVersion) - "", ); } - out.push_str(""); out.push_str(""); } ItemShape::CalendarItem => { @@ -216,6 +222,7 @@ const CALENDAR_FIELDS: &[&str] = &[ "calendar:IsOnlineMeeting", "calendar:MeetingWorkspaceUrl", "calendar:NetShowUrl", + "calendar:MyResponseType", "item:ReminderIsSet", "item:ReminderMinutesBeforeStart", "calendar:Recurrence", @@ -225,6 +232,8 @@ const CALENDAR_FIELDS: &[&str] = &[ "item:Subject", "item:Body", "item:Categories", + "item:Sensitivity", + "item:Attachments", "item:DateTimeCreated", "item:LastModifiedTime", "item:ParentFolderId", @@ -302,6 +311,7 @@ mod tests { assert!(body.contains("folder:FolderClass")); assert!(body.contains("folder:DisplayName")); assert!(body.contains("folder:ParentFolderId")); + assert!(body.contains("PropertyTag=\"0x10F4\" PropertyType=\"Boolean\"")); } #[test] @@ -318,7 +328,7 @@ mod tests { let folder = FolderId::new("FID", "FCK"); let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 50); assert!(body.contains( - "IdOnly" + "" )); } @@ -395,7 +405,6 @@ mod tests { "the extended-property fallback is only used when item:Flag is unavailable" ); assert!(legacy.contains("item:DateTimeReceived")); - assert!(legacy.contains("message:IsReadReceiptRequested")); assert!( modern.contains("\"message:IsRead\"") && !modern.contains("\"item:IsRead\""), "IsRead is a MessageType property; its FieldURI is message:IsRead" @@ -434,6 +443,16 @@ mod tests { assert!(body.contains("STATE")); assert!(body.contains("512")); assert!(body.contains("NormalItems")); + assert!(body.contains("FieldURI=\"item:ItemClass\"")); + } + + #[test] + fn find_item_requests_item_class() { + let folder = FolderId::new("FID", "FCK"); + let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 10); + assert!(body.contains( + "IdOnly" + )); } #[test] @@ -448,6 +467,22 @@ mod tests { ); } + #[test] + fn calendar_shape_requests_attachments_own_response_and_sensitivity() { + let body = get_item_body( + ItemShape::CalendarItem, + &[ItemId::new("X", "")], + ServerVersion::Exchange2007, + ); + for field in [ + "item:Attachments", + "calendar:MyResponseType", + "item:Sensitivity", + ] { + assert!(body.contains(&format!("\"{field}\"")), "{field}"); + } + } + #[test] fn calendar_timezone_fields_gated_on_exchange_2010() { let modern = get_item_body( diff --git a/src/exchange_graph/api.rs b/src/exchange_graph/api.rs index afd8e8f..a5bf5be 100644 --- a/src/exchange_graph/api.rs +++ b/src/exchange_graph/api.rs @@ -26,7 +26,7 @@ pub const EXCEPTION_WINDOW_MAX_DAYS: i64 = 1825; pub const DRIVE_ITEM_SELECT: &str = "id,name,size,folder,file,package,remoteItem,createdDateTime,lastModifiedDateTime"; -pub const MESSAGE_STATE_SELECT: &str = "id,isRead,isDraft,isReadReceiptRequested,flag,categories"; +pub const MESSAGE_STATE_SELECT: &str = "id,isRead,isDraft,flag,categories"; #[derive(Debug, Clone)] pub struct Endpoints { @@ -163,7 +163,7 @@ impl Endpoints { format!( "{}/calendars/{}/calendarView?startDateTime={window_start}&endDateTime={window_end}\ &$filter=type%20eq%20%27occurrence%27&$top={top}\ - &$select=id,type,seriesMasterId,start", + &$select=id,type,seriesMasterId,start,originalStartTimeZone", self.me_or_user(), url_escape(calendar_id) ) diff --git a/src/exchange_graph/calendar_map.rs b/src/exchange_graph/calendar_map.rs index 99e4142..6ea05f7 100644 --- a/src/exchange_graph/calendar_map.rs +++ b/src/exchange_graph/calendar_map.rs @@ -5,9 +5,11 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use chrono::NaiveDate; use serde_json::{Map, Value, json}; use crate::exchange::address::{as_smtp_address, extract_smtp_address}; +use crate::exchange::date::all_day_date; use crate::exchange::jscalendar::{drop_calendar_address_dependents, synthetic_attendee_address}; use crate::exchange::tz::resolve_to_iana; use crate::exchange_graph::error::GraphError; @@ -55,6 +57,7 @@ pub struct ConvertedEvent { pub series_master_id: Option, pub event_type: EventType, pub original_start: Option, + pub zone: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -138,24 +141,32 @@ pub fn convert_event( .or_else(|| fallback_calendar_tz.and_then(windows_or_iana_to_iana)) .filter(|z| z != "Etc/UTC"); - if let Some(start) = start_dt.as_deref() { - let value = match (is_all_day, display_tz.as_deref()) { - (false, Some(tz)) => utc_naive_to_local(start, tz).unwrap_or_else(|| start.to_owned()), - _ => start.to_owned(), - }; - card.insert("start".to_owned(), Value::from(value)); - } - - if let (Some(start), Some(end)) = (start_dt.as_deref(), end_dt.as_deref()) - && let Some(dur) = duration_iso8601(start, end) - { - card.insert("duration".to_owned(), Value::from(dur)); - } - if is_all_day { + let zone = display_tz.as_deref(); + let start = start_dt.as_deref().and_then(|s| all_day_date(s, zone)); + if let Some(start) = start { + card.insert("start".to_owned(), Value::from(all_day_key(start))); + let days = end_dt + .as_deref() + .and_then(|e| all_day_date(e, zone)) + .map_or(1, |end| (end - start).num_days().max(1)); + card.insert("duration".to_owned(), Value::from(format!("P{days}D"))); + } card.insert("showWithoutTime".to_owned(), Value::Bool(true)); } else { - let label = display_tz.unwrap_or_else(|| "Etc/UTC".to_owned()); + if let Some(start) = start_dt.as_deref() { + let value = display_tz + .as_deref() + .and_then(|tz| utc_naive_to_local(start, tz)) + .unwrap_or_else(|| start.to_owned()); + card.insert("start".to_owned(), Value::from(value)); + } + if let (Some(start), Some(end)) = (start_dt.as_deref(), end_dt.as_deref()) + && let Some(dur) = duration_iso8601(start, end) + { + card.insert("duration".to_owned(), Value::from(dur)); + } + let label = display_tz.as_deref().unwrap_or("Etc/UTC"); card.insert("timeZone".to_owned(), Value::from(label)); } @@ -344,9 +355,14 @@ pub fn convert_event( series_master_id, event_type, original_start, + zone: display_tz, }) } +pub fn all_day_key(date: NaiveDate) -> String { + format!("{date}T00:00:00") +} + fn cancelled_occurrence_key(entry: &str, start_time: Option<&str>) -> Option { let date = entry.rsplit('.').next().filter(|d| d.len() == 10)?; if !date.as_bytes().iter().enumerate().all(|(i, b)| match i { @@ -922,6 +938,31 @@ mod tests { assert!(conv.data.get("timeZone").is_none()); } + #[test] + fn all_day_utc_instant_east_of_utc_keeps_its_local_date() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + v["originalStartTimeZone"] = Value::from("Romance Standard Time"); + v["start"]["dateTime"] = Value::from("2026-10-23T22:00:00.0000000"); + v["end"]["dateTime"] = Value::from("2026-10-25T23:00:00.0000000"); + let conv = convert_event(&v, None).unwrap(); + assert_eq!(conv.data["start"], "2026-10-24T00:00:00"); + assert_eq!(conv.data["duration"], "P2D"); + assert_eq!(conv.zone.as_deref(), Some("Europe/Paris")); + } + + #[test] + fn all_day_midnight_is_kept_whatever_the_zone() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + v["originalStartTimeZone"] = Value::from("New Zealand Standard Time"); + v["start"]["dateTime"] = Value::from("2026-11-12T00:00:00.0000000"); + v["end"]["dateTime"] = Value::from("2026-11-13T00:00:00.0000000"); + let conv = convert_event(&v, None).unwrap(); + assert_eq!(conv.data["start"], "2026-11-12T00:00:00"); + assert_eq!(conv.data["duration"], "P1D"); + } + #[test] fn duration_h_and_s_no_minutes_includes_zero_m() { assert_eq!( diff --git a/src/exchange_graph/contact_map.rs b/src/exchange_graph/contact_map.rs index 7e4adfd..7f3d6e3 100644 --- a/src/exchange_graph/contact_map.rs +++ b/src/exchange_graph/contact_map.rs @@ -5,8 +5,10 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use chrono::Datelike; use serde_json::{Map, Value, json}; +use crate::exchange::date::{is_outlook_no_date, nearest_midnight}; use crate::exchange_graph::error::GraphError; #[derive(Debug, Clone)] @@ -373,24 +375,17 @@ fn normalise_utc_datetime(raw: &str) -> String { } fn parse_partial_date(raw: &str) -> Option { - let date_part = raw.split('T').next().unwrap_or(raw); - let mut iter = date_part.split('-'); - let year: u64 = iter.next()?.parse().ok()?; + if is_outlook_no_date(raw) { + return None; + } + let date = nearest_midnight(raw, None)?; let mut obj = Map::new(); obj.insert("@type".to_owned(), Value::from("PartialDate")); - obj.insert("year".to_owned(), Value::from(year)); - if let Some(month_str) = iter.next() - && let Ok(month) = month_str.parse::() - && (1..=12).contains(&month) - { - obj.insert("month".to_owned(), Value::from(month)); - if let Some(day_str) = iter.next() - && let Ok(day) = day_str.parse::() - && (1..=31).contains(&day) - { - obj.insert("day".to_owned(), Value::from(day)); - } + if date.year() > 0 { + obj.insert("year".to_owned(), Value::from(date.year())); } + obj.insert("month".to_owned(), Value::from(date.month())); + obj.insert("day".to_owned(), Value::from(date.day())); Some(Value::Object(obj)) } @@ -578,6 +573,24 @@ mod tests { assert_eq!(ja["organizations/org-1/name"], "アクメ"); } + #[test] + fn birthday_stored_as_local_midnight_in_utc_keeps_its_day() { + for raw in ["1990-04-14T22:00:00Z", "1990-04-15T11:59:00Z", "1990-04-15"] { + let conv = convert_contact(&json!({"id": "X", "birthday": raw})).unwrap(); + assert_eq!( + conv.data["anniversaries"]["anniversary-1"]["date"], + json!({"@type": "PartialDate", "year": 1990, "month": 4, "day": 15}) + ); + } + } + + #[test] + fn outlook_no_date_birthday_is_dropped() { + let c = json!({"id": "X", "birthday": "4501-01-01T00:00:00Z"}); + let conv = convert_contact(&c).unwrap(); + assert!(conv.data.get("anniversaries").is_none()); + } + #[test] fn birthday_emits_partialdate_object() { let c = json!({"id": "X", "birthday": "1980-04-12T00:00:00Z"}); diff --git a/src/exchange_graph/recurrence.rs b/src/exchange_graph/recurrence.rs index 43568eb..9044e98 100644 --- a/src/exchange_graph/recurrence.rs +++ b/src/exchange_graph/recurrence.rs @@ -6,7 +6,7 @@ use serde_json::{Value, json}; -use crate::exchange::date::recurrence_until; +use crate::exchange::date::{is_outlook_no_date, recurrence_until}; use crate::exchange_graph::error::GraphError; pub fn convert_patterned_recurrence(pr: &Value) -> Result { @@ -86,6 +86,7 @@ pub fn convert_patterned_recurrence_rule( if let Some(local) = range .get("endDate") .and_then(Value::as_str) + .filter(|end| !is_outlook_no_date(end)) .and_then(recurrence_until) { rule.insert("until".to_owned(), Value::from(local)); @@ -275,6 +276,16 @@ mod tests { assert_eq!(out["until"], "2026-12-31T23:59:59"); } + #[test] + fn range_end_on_outlook_no_date_has_no_until() { + let rule = convert_patterned_recurrence(&json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "endDate", "startDate": "2026-01-01", "endDate": "4500-12-31"} + })) + .unwrap(); + assert!(rule.get("until").is_none()); + } + #[test] fn range_end_datetime_preserved() { let pr = json!({ diff --git a/src/sync/emailmeta.rs b/src/sync/emailmeta.rs index df991bf..1c1945c 100644 --- a/src/sync/emailmeta.rs +++ b/src/sync/emailmeta.rs @@ -69,7 +69,7 @@ fn add_keywords_header(m: &Message<'_>, keywords: &mut Vec) { } } -fn jmap_keyword(phrase: &str) -> Option { +pub(crate) fn jmap_keyword(phrase: &str) -> Option { let mut keyword = String::with_capacity(phrase.len().min(MAX_KEYWORD_LEN)); let mut separator = false; for ch in phrase.chars() { diff --git a/src/sync/import_exchange_ews/calendar.rs b/src/sync/import_exchange_ews/calendar.rs index 2ea6bda..0cbe804 100644 --- a/src/sync/import_exchange_ews/calendar.rs +++ b/src/sync/import_exchange_ews/calendar.rs @@ -10,6 +10,7 @@ use serde_json::Value; use crate::db::exchange_ews_ids; use crate::error::Error; +use crate::exchange::date::is_outlook_no_date; use crate::exchange_ews::calendar_map::to_jscalendar_with_exceptions; use crate::exchange_ews::parse::{CalendarItemRaw, parse_calendar_item}; use crate::exchange_ews::types::{CalendarItemType, ItemId}; @@ -124,7 +125,8 @@ fn reconcile_one( } return Ok(()); } - let parsed = parse_calendar_item(&msg.inner_xml).map_err(Error::from)?; + let parsed = parse_calendar_item(&msg.inner_xml) + .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; if parsed.id.id.is_empty() { counts.failed += 1; return Ok(()); @@ -132,7 +134,8 @@ fn reconcile_one( if matches!( parsed.calendar_item_type, Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception) - ) { + ) || parsed.start.as_deref().is_some_and(is_outlook_no_date) + { counts.skipped += 1; return Ok(()); } @@ -185,7 +188,7 @@ fn apply_event( counts: &mut TypeCounts, ) -> Result<(), Error> { let modified_full = fetch_modified_occurrences(ctx, raw); - let mut built = to_jscalendar_with_exceptions(raw, &modified_full); + let mut built = to_jscalendar_with_exceptions(raw, &modified_full, ctx.owner_smtp); attach_calendar_links(conn, ctx, raw, &mut built.data)?; let calendar_ids = serde_json::json!([local_folder_id]).to_string(); let data = built.data.to_string(); diff --git a/src/sync/import_exchange_ews/contacts.rs b/src/sync/import_exchange_ews/contacts.rs index 726e574..48aac4a 100644 --- a/src/sync/import_exchange_ews/contacts.rs +++ b/src/sync/import_exchange_ews/contacts.rs @@ -112,7 +112,8 @@ fn reconcile_one( } return Ok(()); } - let parsed = parse_contact_item(&msg.inner_xml).map_err(Error::from)?; + let parsed = parse_contact_item(&msg.inner_xml) + .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; if parsed.id.id.is_empty() { counts.failed += 1; return Ok(()); diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs index 455fb27..584b5d0 100644 --- a/src/sync/import_exchange_ews/coordinator.rs +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -167,6 +167,8 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result o, Err(e) => { ctx.logger.warn(&format!( @@ -285,6 +289,12 @@ fn estimate_items( continue; } }; + if item_type == exchange_ews_ids::EMAIL { + // As the import does: contacts, appointments and the like in a + // mail folder are not stored, so they are not counted either. + let skipped = messages::drop_non_mail_items(&mut outcome.items); + counts.skipped += skipped.iter().map(|(_, n)| n).sum::(); + } let local = exchange_ews_ids::items_in_folder(conn, ctx.source_id, item_type, &fid.id) .unwrap_or_default(); let (new, bytes) = items::new_count_and_bytes(&outcome, &local); diff --git a/src/sync/import_exchange_ews/folders.rs b/src/sync/import_exchange_ews/folders.rs index 0c25831..ed6687f 100644 --- a/src/sync/import_exchange_ews/folders.rs +++ b/src/sync/import_exchange_ews/folders.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use rusqlite::{Connection, params}; @@ -13,12 +13,12 @@ use crate::error::Error; use crate::exchange_ews::EwsClient; use crate::exchange_ews::error::EwsError; use crate::exchange_ews::parse::{ - FolderEntry, parse_find_folder_response, parse_folder_inner, parse_get_folder_response, - parse_response_messages, + FolderEntry, parse_find_folder_response, parse_find_item_response, parse_folder_inner, + parse_get_folder_response, parse_response_messages, }; use crate::exchange_ews::types::{DistinguishedFolderId, FolderClass, MailboxKind}; use crate::exchange_ews::xml::{ - FolderRef, FolderShape, Traversal, find_folder_body, get_folder_body, + FolderRef, FolderShape, Traversal, find_folder_body, find_item_body, get_folder_body, }; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::sync::TypeCounts; @@ -52,24 +52,33 @@ pub fn plan_folders( if logger.enabled(LEVEL_PROGRESS) { eprintln!("FindFolder returned {} folders", parsed.folders.len()); } + let folders = drop_hidden_subtrees(parsed.folders); + let parents: HashMap = folders + .iter() + .filter_map(|f| Some((f.folder_id.id.clone(), f.parent_id.clone()?))) + .collect(); let mut mail = Vec::new(); let mut calendar = Vec::new(); let mut contacts = Vec::new(); - for folder in parsed.folders { - let class = FolderClass::from_ipf(&folder.folder_class); - if class == FolderClass::Mail - && FolderClass::is_mail_fallback(&folder.folder_class) - && logger.enabled(LEVEL_DEFAULT) - { - let shown = if folder.folder_class.trim().is_empty() { - "no" - } else { - "unrecognized" - }; - eprintln!( - "warning: EWS folder {:?} has {shown} FolderClass ({:?}); importing as mail", - folder.display_name, folder.folder_class - ); + for folder in folders { + let mut class = FolderClass::from_ipf(&folder.folder_class); + if class == FolderClass::Mail && FolderClass::is_mail_fallback(&folder.folder_class) { + let classless = folder.folder_class.trim().is_empty(); + if classless && !matches!(mailbox_kind, MailboxKind::Archive) { + class = kind_from_items(client, url, &folder, logger); + } + if logger.enabled(LEVEL_DEFAULT) { + let shown = if classless { "no" } else { "unrecognized" }; + let kind = match class { + FolderClass::Calendar => "calendar", + FolderClass::Contacts => "contacts", + FolderClass::Mail | FolderClass::Skipped => "mail", + }; + eprintln!( + "warning: EWS folder {:?} has {shown} FolderClass ({:?}); importing as {kind}", + folder.display_name, folder.folder_class + ); + } } match (mailbox_kind, class) { (MailboxKind::Archive, FolderClass::Mail) => { @@ -98,6 +107,7 @@ pub fn plan_folders( (_, FolderClass::Skipped) => {} } } + reparent_under_mail_ancestors(&mut mail, &parents); let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, mailbox, logger)?; let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind, mailbox)?; Ok(FolderPlan { @@ -109,6 +119,92 @@ pub fn plan_folders( }) } +const CLASS_SAMPLE_SIZE: u32 = 10; + +fn kind_from_items( + client: &EwsClient, + url: &str, + folder: &FolderEntry, + logger: Logger, +) -> FolderClass { + let request = find_item_body( + FolderRef::Concrete(&folder.folder_id), + Traversal::Shallow, + 0, + CLASS_SAMPLE_SIZE, + ); + let sampled = client + .call(url, "FindItem", &request) + .and_then(|resp| parse_find_item_response(&resp.body)); + match sampled { + Ok(found) => FolderClass::from_item_classes( + found + .items + .iter() + .filter_map(|item| item.item_class.as_deref()), + ), + Err(e) => { + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "FindItem sample of {:?} failed: {e}; treating it as mail", + folder.display_name + ); + } + FolderClass::Mail + } + } +} + +fn drop_hidden_subtrees(mut folders: Vec) -> Vec { + if !folders.iter().any(|f| f.is_hidden) { + return folders; + } + let by_id: HashMap<&str, &FolderEntry> = folders + .iter() + .map(|f| (f.folder_id.id.as_str(), f)) + .collect(); + let hidden: HashSet = folders + .iter() + .filter(|f| { + let mut current = Some(*f); + for _ in 0..=by_id.len() { + let Some(folder) = current else { + return false; + }; + if folder.is_hidden { + return true; + } + current = folder + .parent_id + .as_deref() + .and_then(|p| by_id.get(p).copied()); + } + false + }) + .map(|f| f.folder_id.id.clone()) + .collect(); + folders.retain(|f| !hidden.contains(&f.folder_id.id)); + folders +} + +fn reparent_under_mail_ancestors(mail: &mut [ClassifiedFolder], parents: &HashMap) { + let mail_ids: HashSet = mail.iter().map(|f| f.folder.folder_id.id.clone()).collect(); + for folder in mail.iter_mut() { + let mut parent = folder.folder.parent_id.as_deref(); + for _ in 0..=parents.len() { + match parent { + Some(p) if !mail_ids.contains(p) => match parents.get(p) { + Some(grandparent) => parent = Some(grandparent), + None => break, + }, + _ => break, + } + } + let parent = parent.map(str::to_owned); + folder.folder.parent_id = parent; + } +} + fn resolve_well_known_roles( client: &EwsClient, url: &str, @@ -216,14 +312,10 @@ pub fn reconcile( exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::ADDRESS_BOOK) .map_err(|e| Error::Partial(e.to_string()))?; - let mut id_to_local: HashMap = HashMap::new(); - for (id, row) in local_mailbox + let mut mailbox_local_ids: HashMap = local_mailbox .iter() - .chain(local_calendar.iter()) - .chain(local_addressbook.iter()) - { - id_to_local.insert(id.clone(), row.local_id); - } + .map(|(id, row)| (id.clone(), row.local_id)) + .collect(); let ordered_mail = order_by_parent(&plan.mail); let ordered_calendar = order_by_parent(&plan.calendar); @@ -242,19 +334,17 @@ pub fn reconcile( let mut ctx = UpsertCtx { conn, source_id, - id_to_local: &mut id_to_local, local: &local_mailbox, counts: &mut *counts.mailbox, root_folder_id: &plan.root_folder_id, }; - upsert_mailbox(&mut ctx, folder, role)?; + upsert_mailbox(&mut ctx, &mut mailbox_local_ids, folder, role)?; } for folder in &ordered_calendar { server_ids_calendar.push(folder.folder.folder_id.id.clone()); let mut ctx = UpsertCtx { conn, source_id, - id_to_local: &mut id_to_local, local: &local_calendar, counts: &mut *counts.calendar, root_folder_id: &plan.root_folder_id, @@ -266,7 +356,6 @@ pub fn reconcile( let mut ctx = UpsertCtx { conn, source_id, - id_to_local: &mut id_to_local, local: &local_addressbook, counts: &mut *counts.addressbook, root_folder_id: &plan.root_folder_id, @@ -369,7 +458,6 @@ fn order_by_parent(folders: &[ClassifiedFolder]) -> Vec<&ClassifiedFolder> { struct UpsertCtx<'a> { conn: &'a mut Connection, source_id: i64, - id_to_local: &'a mut HashMap, local: &'a HashMap, counts: &'a mut TypeCounts, root_folder_id: &'a str, @@ -377,10 +465,11 @@ struct UpsertCtx<'a> { fn upsert_mailbox( ctx: &mut UpsertCtx<'_>, + mailbox_local_ids: &mut HashMap, folder: &ClassifiedFolder, role: Option<&'static str>, ) -> Result<(), Error> { - let parent_local = parent_local_id(folder, ctx.id_to_local, ctx.root_folder_id); + let parent_local = parent_local_id(folder, mailbox_local_ids, ctx.root_folder_id); let tx = ctx .conn .unchecked_transaction() @@ -428,8 +517,7 @@ fn upsert_mailbox( new_id }; tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - ctx.id_to_local - .insert(folder.folder.folder_id.id.clone(), local_id); + mailbox_local_ids.insert(folder.folder.folder_id.id.clone(), local_id); Ok(()) } @@ -439,7 +527,7 @@ fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result .unchecked_transaction() .map_err(|e| Error::Partial(e.to_string()))?; let existing = ctx.local.get(&folder.folder.folder_id.id); - let local_id = if let Some(row) = existing { + if let Some(row) = existing { tx.execute( "UPDATE calendars SET name = ?1 WHERE id = ?2", params![folder.folder.display_name, row.local_id], @@ -454,7 +542,6 @@ fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result ) .map_err(|e| Error::Partial(e.to_string()))?; ctx.counts.fetched += 1; - row.local_id } else { tx.execute( "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible) \ @@ -474,11 +561,8 @@ fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result ) .map_err(|e| Error::Partial(e.to_string()))?; ctx.counts.created += 1; - new_id - }; + } tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - ctx.id_to_local - .insert(folder.folder.folder_id.id.clone(), local_id); Ok(()) } @@ -488,7 +572,7 @@ fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Re .unchecked_transaction() .map_err(|e| Error::Partial(e.to_string()))?; let existing = ctx.local.get(&folder.folder.folder_id.id); - let local_id = if let Some(row) = existing { + if let Some(row) = existing { tx.execute( "UPDATE address_books SET name = ?1 WHERE id = ?2", params![folder.folder.display_name, row.local_id], @@ -503,7 +587,6 @@ fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Re ) .map_err(|e| Error::Partial(e.to_string()))?; ctx.counts.fetched += 1; - row.local_id } else { tx.execute( "INSERT INTO address_books (name, sort_order, is_subscribed) \ @@ -523,24 +606,21 @@ fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Re ) .map_err(|e| Error::Partial(e.to_string()))?; ctx.counts.created += 1; - new_id - }; + } tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - ctx.id_to_local - .insert(folder.folder.folder_id.id.clone(), local_id); Ok(()) } fn parent_local_id( folder: &ClassifiedFolder, - id_to_local: &HashMap, + mailbox_local_ids: &HashMap, root_folder_id: &str, ) -> Option { let parent_id = folder.folder.parent_id.as_deref()?; if parent_id == root_folder_id || parent_id.is_empty() { return None; } - id_to_local.get(parent_id).copied() + mailbox_local_ids.get(parent_id).copied() } fn delete_vanished( @@ -735,6 +815,177 @@ mod tests { assert_eq!(orphan_idmap, 0, "both idmap rows must be removed"); } + fn entry(id: &str, parent: &str, class: &str, hidden: bool) -> FolderEntry { + FolderEntry { + folder_id: crate::exchange_ews::types::FolderId { + id: id.to_owned(), + change_key: "CK".to_owned(), + }, + parent_id: Some(parent.to_owned()), + display_name: id.to_owned(), + folder_class: class.to_owned(), + is_hidden: hidden, + ..FolderEntry::default() + } + } + + fn classified(id: &str, parent: &str, kind: FolderClass) -> ClassifiedFolder { + ClassifiedFolder { + folder: entry(id, parent, "", false), + kind, + } + } + + #[test] + fn hidden_folders_are_dropped_with_their_descendants() { + let folders = vec![ + entry("CONTACTS", "ROOT", "IPF.Contact", false), + entry("RECIPIENT_CACHE", "CONTACTS", "IPF.Contact", true), + entry("YAMMER", "ROOT", "IPF.Note", true), + entry("YAMMER_FEEDS", "YAMMER", "IPF.Note", false), + entry("YAMMER_DEEP", "YAMMER_FEEDS", "IPF.Note", false), + entry("INBOX", "ROOT", "IPF.Note", false), + ]; + let kept: Vec = drop_hidden_subtrees(folders) + .into_iter() + .map(|f| f.folder_id.id) + .collect(); + assert_eq!(kept, ["CONTACTS", "INBOX"]); + } + + #[test] + fn mail_folder_is_filed_under_its_nearest_mail_ancestor() { + let parents: HashMap = [ + ("PROJECTS", "ROOT"), + ("PCAL", "PROJECTS"), + ("SUB", "PCAL"), + ("CAL", "ROOT"), + ("NOTES", "CAL"), + ("INBOX", "ROOT"), + ("A", "B"), + ("B", "A"), + ("LOOP", "A"), + ] + .into_iter() + .map(|(id, parent)| (id.to_owned(), parent.to_owned())) + .collect(); + let mut mail = vec![ + classified("PROJECTS", "ROOT", FolderClass::Mail), + classified("SUB", "PCAL", FolderClass::Mail), + classified("NOTES", "CAL", FolderClass::Mail), + classified("INBOX", "ROOT", FolderClass::Mail), + classified("LOOP", "A", FolderClass::Mail), + ]; + reparent_under_mail_ancestors(&mut mail, &parents); + let parent_of = |id: &str| { + mail.iter() + .find(|f| f.folder.folder_id.id == id) + .and_then(|f| f.folder.parent_id.clone()) + }; + assert_eq!(parent_of("SUB").as_deref(), Some("PROJECTS")); + assert_eq!(parent_of("NOTES").as_deref(), Some("ROOT")); + assert_eq!(parent_of("INBOX").as_deref(), Some("ROOT")); + assert_ne!(parent_of("LOOP").as_deref(), Some("PROJECTS")); + } + + #[test] + fn classless_folder_kind_comes_from_its_items() { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let _m = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", "text/xml; charset=utf-8") + .with_body( + "\ + \ + NoError\ + \ + IPM.Contact\ + \ + ", + ) + .create(); + let client = EwsClient::new( + crate::jmap::http::Auth::Bearer { + token: "t".to_owned(), + }, + crate::jmap::http::RetryPolicy::new(0), + crate::net::CertOverride::none(), + ); + let folder = entry("LOCAL", "ROOT", "", false); + assert_eq!( + kind_from_items(&client, &url, &folder, Logger::new(LEVEL_DEFAULT)), + FolderClass::Contacts + ); + } + + #[test] + fn rerun_does_not_take_a_calendar_local_id_as_mailbox_parent() { + let mut conn = rusqlite::Connection::open_in_memory().unwrap(); + crate::db::init::apply_schema(&conn).unwrap(); + let sid = crate::db::sources::upsert_source( + &conn, + &crate::db::sources::SourceKey { + kind: "exchange_ews".to_owned(), + session_url: "https://x/EWS/Exchange.asmx".to_owned(), + account_id: "u@d".to_owned(), + }, + Some("u"), + "u@d", + ) + .unwrap(); + let plan = FolderPlan { + mail: vec![ + classified("INBOX", "ROOT", FolderClass::Mail), + classified("NOTES", "CAL", FolderClass::Mail), + ], + calendar: vec![ + classified("CAL", "ROOT", FolderClass::Calendar), + classified("CAL2", "ROOT", FolderClass::Calendar), + ], + contacts: vec![], + well_known_roles: HashMap::new(), + root_folder_id: "ROOT".to_owned(), + }; + for _ in 0..2 { + let mut counts: [TypeCounts; 6] = Default::default(); + let [ + mailbox, + calendar, + addressbook, + email, + calendar_event, + contact, + ] = &mut counts; + reconcile( + &mut conn, + sid, + &plan, + &mut ReconcileCounts { + mailbox, + calendar, + addressbook, + email, + calendar_event, + contact, + }, + Logger::new(LEVEL_DEFAULT), + ) + .unwrap(); + let parent: Option = conn + .query_row( + "SELECT parent_id FROM mailboxes WHERE name = 'NOTES'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(parent, None); + } + } + fn row(id: &str, parent_ews_id: &str, local_id: i64) -> exchange_ews_ids::FolderRow { exchange_ews_ids::FolderRow { item_id: id.to_owned(), diff --git a/src/sync/import_exchange_ews/items.rs b/src/sync/import_exchange_ews/items.rs index 39b3f2c..5806c45 100644 --- a/src/sync/import_exchange_ews/items.rs +++ b/src/sync/import_exchange_ews/items.rs @@ -23,6 +23,7 @@ pub struct ItemRunCtx<'a> { pub client: &'a EwsClient, pub url: &'a str, pub source_id: i64, + pub owner_smtp: Option<&'a str>, pub batch_size: usize, /// Byte cap for one GetItem batch, by `item:Size`; see `sync::batch`. pub batch_bytes: u64, @@ -44,6 +45,7 @@ fn shrink_sync_batch(current: u32) -> u32 { pub struct EnumeratedItem { pub element: String, pub id: ItemId, + pub item_class: Option, } #[derive(Debug, Clone)] @@ -120,8 +122,21 @@ fn try_sync_folder_items( let parsed = parse_sync_folder_items_response(&resp.body)?; for change in parsed.changes { match change { - SyncChange::Create { id, element } | SyncChange::Update { id, element } => { - items.push(EnumeratedItem { element, id }); + SyncChange::Create { + id, + element, + item_class, + } + | SyncChange::Update { + id, + element, + item_class, + } => { + items.push(EnumeratedItem { + element, + id, + item_class, + }); } SyncChange::Delete { id } => { deletions.push(id.id); @@ -130,6 +145,7 @@ fn try_sync_folder_items( items.push(EnumeratedItem { element: "Message".to_owned(), id, + item_class: None, }); } } @@ -187,6 +203,7 @@ fn enumerate_via_find_item( items.push(EnumeratedItem { element: entry.element, id: entry.id, + item_class: entry.item_class, }); } if !parsed.more { @@ -435,7 +452,13 @@ where let outcome = get_item_batches(ctx, shape, win).map_err(Error::from)?; failed_items = failed_items.saturating_add(outcome.failed_items); for msg in outcome.messages { - on_message(msg)?; + if let Err(e) = on_message(msg) { + if e.aborts_run() { + return Err(e); + } + ctx.logger.warn(&format!("EWS item not imported: {e}")); + failed_items = failed_items.saturating_add(1); + } } } Ok(failed_items) @@ -476,6 +499,87 @@ pub fn delete_vanished( #[cfg(test)] mod tests { use super::*; + use crate::jmap::http::{Auth, RetryPolicy}; + use crate::logging::LEVEL_DEFAULT; + + const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ + xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ + xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; + + fn fetch_three(fail_with: fn() -> Error) -> (Result, Vec) { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let messages: String = ["A", "B", "C"] + .iter() + .map(|id| { + format!( + "\ + NoError\ + \ + " + ) + }) + .collect(); + let _m = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", "text/xml; charset=utf-8") + .with_body(format!( + "\ + {messages}" + )) + .create(); + let client = EwsClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + crate::net::CertOverride::none(), + ); + let ctx = ItemRunCtx { + client: &client, + url: &url, + source_id: 1, + owner_smtp: None, + batch_size: 10, + batch_bytes: u64::MAX, + attachment_batch: 10, + connections: 1, + use_syncfolderitems: false, + sync_batch: SYNC_BATCH_MAX, + logger: Logger::new(LEVEL_DEFAULT), + }; + let ids: Vec = ["A", "B", "C"] + .iter() + .map(|id| ItemId::new(*id, "K")) + .collect(); + let mut seen = Vec::new(); + let result = + for_each_fetched_item(&ctx, ItemShape::Message, &ids, &HashMap::new(), |msg| { + let parsed = crate::exchange_ews::parse::parse_message_item(&msg.inner_xml) + .map_err(|e| Error::Partial(e.to_string()))?; + seen.push(parsed.id.id.clone()); + if parsed.id.id == "A" { + return Err(fail_with()); + } + Ok(()) + }); + (result, seen) + } + + #[test] + fn per_item_failure_does_not_abort_the_folder() { + let (result, seen) = fetch_three(|| Error::Partial("UNIQUE constraint failed".to_owned())); + assert_eq!(result.unwrap(), 1); + assert_eq!(seen, ["A", "B", "C"]); + } + + #[test] + fn run_aborting_failure_still_stops() { + let (result, seen) = fetch_three(|| Error::Connection("gone".to_owned())); + assert!(result.is_err()); + assert_eq!(seen, ["A"]); + } #[test] fn sync_batch_shrinks_by_half_down_to_floor() { @@ -493,6 +597,7 @@ mod tests { items: vec![EnumeratedItem { element: "Message".to_owned(), id: ItemId::new("A", "ck-2"), + item_class: None, }], sizes: HashMap::new(), mode: EnumerationMode::Delta { @@ -533,14 +638,17 @@ mod tests { EnumeratedItem { element: "Message".to_owned(), id: ItemId::new("A", "ck-1"), + item_class: None, }, EnumeratedItem { element: "Message".to_owned(), id: ItemId::new("B", "ck-1"), + item_class: None, }, EnumeratedItem { element: "Message".to_owned(), id: ItemId::new("C", "ck-2"), + item_class: None, }, ]; let local = vec![ diff --git a/src/sync/import_exchange_ews/messages.rs b/src/sync/import_exchange_ews/messages.rs index d49bee8..51b0f23 100644 --- a/src/sync/import_exchange_ews/messages.rs +++ b/src/sync/import_exchange_ews/messages.rs @@ -5,6 +5,8 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::fmt::Write; + use encodify::base64::LENIENT; use rusqlite::{Connection, params}; use serde_json::{Value, json}; @@ -12,16 +14,17 @@ use serde_json::{Value, json}; use crate::db::{blobs, exchange_ews_ids}; use crate::error::Error; use crate::exchange_ews::parse::{MessageItem, parse_message_item}; -use crate::exchange_ews::types::ItemId; +use crate::exchange_ews::types::{ItemId, is_non_mail_item_class}; use crate::exchange_ews::xml::ItemShape; -use crate::logging::LEVEL_PROGRESS; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS}; use crate::sync::TypeCounts; -use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::emailmeta::{email_meta_from_blob, jmap_keyword}; use crate::sync::keys::index_to_json; use super::folders::FolderPlan; use super::items::{ - EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for, + EnumeratedItem, EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, + for_each_fetched_item, plan_for, }; pub fn reconcile_all( @@ -76,7 +79,22 @@ fn reconcile_one_folder( &folder.id, ) .map_err(|e| Error::Partial(e.to_string()))?; - let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; + let mut outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; + let skipped = drop_non_mail_items(&mut outcome.items); + if !skipped.is_empty() { + let total: u64 = skipped.iter().map(|(_, n)| n).sum(); + counts.skipped += total; + if ctx.logger.enabled(LEVEL_DEFAULT) { + let mut summary = String::new(); + for (class, n) in &skipped { + if !summary.is_empty() { + summary.push_str(", "); + } + let _ = write!(summary, "{class}={n}"); + } + eprintln!("EWS folder {folder_name:?}: skipped {total} non-mail items ({summary})"); + } + } let local = exchange_ews_ids::items_in_folder(conn, ctx.source_id, exchange_ews_ids::EMAIL, &folder.id) .map_err(|e| Error::Partial(e.to_string()))?; @@ -113,7 +131,8 @@ fn reconcile_one_folder( } return Ok(()); } - let parsed = parse_message_item(&msg.inner_xml).map_err(Error::from)?; + let parsed = parse_message_item(&msg.inner_xml) + .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; if parsed.id.id.is_empty() { counts.failed += 1; return Ok(()); @@ -247,6 +266,30 @@ fn apply_message( Ok(()) } +/// Drops the non-mail items a mail folder can hold (contacts, appointments, +/// tasks, notes, journal entries) and returns how many of each class went. +pub(super) fn drop_non_mail_items(items: &mut Vec) -> Vec<(String, u64)> { + let mut skipped: Vec<(String, u64)> = Vec::new(); + items.retain(|item| { + let Some(class) = item + .item_class + .as_deref() + .filter(|c| is_non_mail_item_class(c)) + else { + return true; + }; + match skipped + .iter_mut() + .find(|(c, _)| c.eq_ignore_ascii_case(class)) + { + Some((_, n)) => *n += 1, + None => skipped.push((class.to_owned(), 1)), + } + false + }); + skipped +} + fn keyword_array(item: &MessageItem) -> String { let mut kws: Vec = Vec::new(); if matches!(item.is_read, Some(true)) { @@ -255,14 +298,13 @@ fn keyword_array(item: &MessageItem) -> String { if matches!(item.is_draft, Some(true)) { kws.push("$draft".to_owned()); } - if matches!(item.is_read_receipt_requested, Some(true)) { - kws.push("$notified".to_owned()); - } if matches!(item.flag_status.as_deref(), Some("Flagged")) { kws.push("$flagged".to_owned()); } - for cat in &item.categories { - kws.push(cat.to_ascii_lowercase()); + for keyword in item.categories.iter().filter_map(|cat| jmap_keyword(cat)) { + if !kws.contains(&keyword) { + kws.push(keyword); + } } let value: Value = Value::Array(kws.into_iter().map(Value::String).collect()); value.to_string() @@ -270,8 +312,37 @@ fn keyword_array(item: &MessageItem) -> String { #[cfg(test)] mod tests { - use super::keyword_array; + use super::{drop_non_mail_items, keyword_array}; use crate::exchange_ews::parse::MessageItem; + use crate::exchange_ews::types::ItemId; + use crate::sync::import_exchange_ews::items::EnumeratedItem; + + #[test] + fn non_mail_items_are_dropped_and_counted_per_class() { + let mut items: Vec = [ + ("M1", Some("IPM.Note")), + ("C1", Some("IPM.Contact")), + ("R1", Some("IPM.Schedule.Meeting.Request")), + ("C2", Some("IPM.Contact")), + ("T1", Some("IPM.Task")), + ("P1", Some("IPM.Post")), + ("X1", None), + ] + .into_iter() + .map(|(id, class)| EnumeratedItem { + element: "Item".to_owned(), + id: ItemId::new(id, "K"), + item_class: class.map(str::to_owned), + }) + .collect(); + let skipped = drop_non_mail_items(&mut items); + let kept: Vec<&str> = items.iter().map(|i| i.id.id.as_str()).collect(); + assert_eq!(kept, ["M1", "R1", "P1", "X1"]); + assert_eq!( + skipped, + [("IPM.Contact".to_owned(), 2), ("IPM.Task".to_owned(), 1)] + ); + } #[test] fn read_draft_flagged_and_categories_map_to_jmap_keywords() { @@ -283,11 +354,25 @@ mod tests { ..MessageItem::default() }; let kws = keyword_array(&item); - for expected in ["$seen", "$draft", "$flagged", "red category", "vip"] { + for expected in ["$seen", "$draft", "$flagged", "red_category", "vip"] { assert!(kws.contains(expected), "missing {expected} in {kws}"); } } + #[test] + fn categories_become_valid_deduplicated_keywords() { + let item = MessageItem { + categories: vec![ + "Catégorie Rouge".to_owned(), + "catégorie rouge".to_owned(), + "$seen".to_owned(), + "( )".to_owned(), + ], + ..MessageItem::default() + }; + assert_eq!(keyword_array(&item), r#"["cat_gorie_rouge","seen"]"#); + } + #[test] fn unread_unflagged_message_has_no_seen_or_flagged() { let item = MessageItem { diff --git a/src/sync/import_exchange_graph/calendar.rs b/src/sync/import_exchange_graph/calendar.rs index 02bfb94..4b3683e 100644 --- a/src/sync/import_exchange_graph/calendar.rs +++ b/src/sync/import_exchange_graph/calendar.rs @@ -12,10 +12,12 @@ use serde_json::{Value, json}; use crate::db::exchange_graph_ids; use crate::error::Error; +use crate::exchange::date::{all_day_date, is_outlook_no_date}; use crate::exchange::jscalendar::override_patch_from_event; use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC}; use crate::exchange_graph::calendar_map::{ - ConvertedEvent, EventType, classify_event_type, convert_event, + ConvertedEvent, EventType, all_day_key, classify_event_type, convert_event, + windows_or_iana_to_iana, }; use crate::exchange_graph::error::GraphError; use crate::exchange_graph::types::EventBodyFormat; @@ -28,6 +30,11 @@ use super::folders::CalendarFolder; pub type EventAttachment = (String, String, Vec); +struct Occurrence { + start: String, + zone: Option, +} + fn fetch_event_attachments( client: &crate::exchange_graph::client::GraphClient, endpoints: &crate::exchange_graph::api::Endpoints, @@ -132,6 +139,7 @@ pub fn reconcile_all( let mut attachments_by_id: HashMap> = HashMap::new(); for (graph_id, result, attachments) in fetched { match result { + Ok(raw) if starts_on_outlook_no_date(&raw) => counts.skipped += 1, Ok(raw) => match convert_event(&raw, None) { Ok(c) => { attachments_by_id.insert(graph_id.clone(), attachments); @@ -209,6 +217,13 @@ fn fetch_events( out } +fn starts_on_outlook_no_date(raw: &Value) -> bool { + raw.get("start") + .and_then(|s| s.get("dateTime")) + .and_then(Value::as_str) + .is_some_and(is_outlook_no_date) +} + fn exception_windows(years: i32) -> Vec<(String, String)> { let today = chrono::Utc::now().date_naive(); let span = chrono::Duration::days( @@ -313,11 +328,7 @@ fn apply_series_expansion( let Some(raw_key) = ex.original_start.as_deref() else { continue; }; - let tz = card - .get("timeZone") - .and_then(Value::as_str) - .map(str::to_owned); - let key = normalise_override_key(raw_key, tz.as_deref()); + let key = occurrence_key(&card, raw_key, ex.zone.as_deref()); if let Some(map) = card.as_object_mut() { let entry = map .entry("recurrenceOverrides".to_owned()) @@ -369,7 +380,7 @@ fn apply_series_expansion( fn cancelled_occurrence_keys( card: &Value, master_id: &str, - occurrences: &HashMap>, + occurrences: &HashMap>, exceptions: &[&ConvertedEvent], windows: &[(String, String)], ) -> Vec { @@ -391,16 +402,13 @@ fn cancelled_occurrence_keys( else { return Vec::new(); }; - let tz = card.get("timeZone").and_then(Value::as_str); - let mut actual: std::collections::HashSet = std::collections::HashSet::new(); - for utc in occurrences.get(master_id).into_iter().flatten() { - actual.insert(local_date_of(utc, tz)); + for occ in occurrences.get(master_id).into_iter().flatten() { + actual.insert(local_date_of(card, &occ.start, occ.zone.as_deref())); } for ex in exceptions { if let Some(raw) = ex.original_start.as_deref() { - let key = normalise_override_key(raw, tz); - actual.insert(key.split('T').next().unwrap_or(&key).to_owned()); + actual.insert(local_date_of(card, raw, ex.zone.as_deref())); } } @@ -427,17 +435,30 @@ fn cancelled_occurrence_keys( out } -fn local_date_of(utc: &str, tz: Option<&str>) -> String { - let key = normalise_override_key(utc, tz); - key.split('T').next().unwrap_or(&key).to_owned() +fn local_date_of(master: &Value, utc: &str, zone: Option<&str>) -> String { + let mut key = occurrence_key(master, utc, zone); + if let Some((date, _)) = key.split_once('T') { + let len = date.len(); + key.truncate(len); + } + key +} + +fn occurrence_key(master: &Value, utc: &str, zone: Option<&str>) -> String { + if master.get("showWithoutTime").and_then(Value::as_bool) == Some(true) + && let Some(date) = all_day_date(utc, zone) + { + return all_day_key(date); + } + normalise_override_key(utc, master.get("timeZone").and_then(Value::as_str)) } fn fetch_calendar_occurrences( ctx: &GraphCoordinator<'_>, calendar_id: &str, counts: &mut TypeCounts, -) -> HashMap> { - let mut out: HashMap> = HashMap::new(); +) -> HashMap> { + let mut out: HashMap> = HashMap::new(); for (from, to) in exception_windows(ctx.exception_window_years) { let url = ctx .endpoints @@ -455,9 +476,14 @@ fn fetch_calendar_occurrences( else { continue; }; - out.entry(master.to_owned()) - .or_default() - .push(start.to_owned()); + let zone = raw + .get("originalStartTimeZone") + .and_then(Value::as_str) + .and_then(windows_or_iana_to_iana); + out.entry(master.to_owned()).or_default().push(Occurrence { + start: start.to_owned(), + zone, + }); } } Err(e) => { @@ -651,6 +677,56 @@ fn delete_vanished( mod tests { use super::*; + fn all_day_daily_master() -> Value { + json!({ + "@type": "Event", + "start": "2026-11-12T00:00:00", + "duration": "P1D", + "showWithoutTime": true, + "recurrenceRule": {"@type": "RecurrenceRule", "frequency": "daily", "count": 3} + }) + } + + #[test] + fn all_day_override_key_takes_the_date_in_the_event_zone() { + let master = all_day_daily_master(); + let paris = Some("Europe/Paris"); + assert_eq!( + occurrence_key(&master, "2026-11-12T23:00:00Z", paris), + "2026-11-13T00:00:00" + ); + assert_eq!( + occurrence_key(&master, "2026-11-13T00:00:00Z", paris), + "2026-11-13T00:00:00" + ); + assert_eq!( + local_date_of(&master, "2026-11-13T23:00:00.0000000", paris), + "2026-11-14" + ); + } + + #[test] + fn all_day_occurrences_east_of_utc_are_not_reported_cancelled() { + let master = all_day_daily_master(); + let zone = Some("Europe/Paris".to_owned()); + let occurrences = HashMap::from([( + "M".to_owned(), + ["2026-11-11T23:00:00.0000000", "2026-11-13T23:00:00.0000000"] + .into_iter() + .map(|start| Occurrence { + start: start.to_owned(), + zone: zone.clone(), + }) + .collect(), + )]); + let windows = vec![( + "2026-11-01T00:00:00".to_owned(), + "2026-11-30T00:00:00".to_owned(), + )]; + let cancelled = cancelled_occurrence_keys(&master, "M", &occurrences, &[], &windows); + assert_eq!(cancelled, vec!["2026-11-13T00:00:00".to_owned()]); + } + #[test] fn exception_windows_never_exceed_the_graph_limit() { for requested in [1, 5, 50] { diff --git a/src/sync/import_exchange_graph/messages.rs b/src/sync/import_exchange_graph/messages.rs index a26fa34..041a834 100644 --- a/src/sync/import_exchange_graph/messages.rs +++ b/src/sync/import_exchange_graph/messages.rs @@ -14,7 +14,7 @@ use crate::exchange_graph::client::Accept; use crate::exchange_graph::error::GraphError; use crate::logging::LEVEL_PROGRESS; use crate::sync::TypeCounts; -use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::emailmeta::{email_meta_from_blob, jmap_keyword}; use crate::sync::import_jmap::pool::Pool; use crate::sync::keys::index_to_json; @@ -188,9 +188,6 @@ fn keyword_array(stub: &Value) -> String { if stub.get("isDraft").and_then(Value::as_bool) == Some(true) { kws.push("$draft".to_owned()); } - if stub.get("isReadReceiptRequested").and_then(Value::as_bool) == Some(true) { - kws.push("$notified".to_owned()); - } if stub .get("flag") .and_then(|f| f.get("flagStatus")) @@ -200,8 +197,14 @@ fn keyword_array(stub: &Value) -> String { kws.push("$flagged".to_owned()); } if let Some(cats) = stub.get("categories").and_then(Value::as_array) { - for cat in cats.iter().filter_map(Value::as_str) { - kws.push(cat.to_ascii_lowercase()); + for keyword in cats + .iter() + .filter_map(Value::as_str) + .filter_map(jmap_keyword) + { + if !kws.contains(&keyword) { + kws.push(keyword); + } } } Value::Array(kws.into_iter().map(Value::String).collect()).to_string() diff --git a/tests/mock_exchange_ews.rs b/tests/mock_exchange_ews.rs index 5362d5e..43cb3e4 100644 --- a/tests/mock_exchange_ews.rs +++ b/tests/mock_exchange_ews.rs @@ -497,14 +497,17 @@ fn coordinator_diff_classifies_new_vanished_changed_unchanged() { EnumeratedItem { element: "Message".into(), id: ItemId::new("A", "ck-1"), + item_class: None, }, EnumeratedItem { element: "Message".into(), id: ItemId::new("B", "ck-2"), + item_class: None, }, EnumeratedItem { element: "Message".into(), id: ItemId::new("C", "ck-1"), + item_class: None, }, ]; let local = vec![ @@ -547,14 +550,17 @@ fn estimate_counts_new_items_and_sums_their_sizes() { EnumeratedItem { element: "Message".into(), id: ItemId::new("A", "ck-1"), + item_class: None, }, EnumeratedItem { element: "Message".into(), id: ItemId::new("B", "ck-1"), + item_class: None, }, EnumeratedItem { element: "Message".into(), id: ItemId::new("C", "ck-1"), + item_class: None, }, ]; let mut sizes = HashMap::new(); @@ -931,6 +937,7 @@ fn for_each_fetched_item_streams_every_id_across_windows() { client: &c, url: &url, source_id: 1, + owner_smtp: None, batch_size: 1, batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, attachment_batch: 1, @@ -982,6 +989,7 @@ fn getitem_batches_are_split_by_bytes_when_sizes_are_known() { client: &c, url: &url, source_id: 1, + owner_smtp: None, batch_size: 10, batch_bytes: 30, attachment_batch: 1, diff --git a/tests/mock_exchange_graph.rs b/tests/mock_exchange_graph.rs index 651f03b..22b2541 100644 --- a/tests/mock_exchange_graph.rs +++ b/tests/mock_exchange_graph.rs @@ -191,7 +191,10 @@ fn pagination_collects_ids_across_three_pages_with_one_empty() { .to_string(); let _m1 = server - .mock("GET", "/me/mailFolders/F1/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/F1/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(page1) @@ -225,7 +228,10 @@ fn pagination_terminates_when_no_next_link_present() { let base = server.url(); let body = json!({"value": [{"id": "A"}, {"id": "B"}]}).to_string(); let _m = server - .mock("GET", "/me/mailFolders/F/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/F/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(body) @@ -253,7 +259,10 @@ fn header_redelivery_is_asserted_via_mockito_match() { .to_string(); let page2 = json!({"value": [{"id": "OnlyOnPageTwo"}]}).to_string(); let _m1 = server - .mock("GET", "/me/mailFolders/F/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/F/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .match_header("prefer", "IdType=\"ImmutableId\"") .match_header("authorization", "Bearer BEARER") .with_status(200) @@ -280,7 +289,10 @@ fn case_sensitive_immutable_id_round_trip() { let id = "AAkAAGFsaWNlAAA="; let body = json!({"value": [{"id": id}]}).to_string(); let _m = server - .mock("GET", "/me/mailFolders/F/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/F/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(body) @@ -673,7 +685,10 @@ fn integration_full_run_mail_only_imports_mime_via_value() { }) .collect(); let _ids = server - .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"value":[{"id":"MSG-1"}]}"#) @@ -774,7 +789,10 @@ fn integration_duplicate_message_id_does_not_abort_run() { }) .collect(); let _ids = server - .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"value":[{"id":"MSG-1"},{"id":"MSG-1"}]}"#) @@ -875,7 +893,10 @@ fn integration_full_run_is_convergent_on_second_invocation() { .create(); } let _ids = server - .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"value":[{"id":"MSG-A"}]}"#) @@ -1396,7 +1417,7 @@ fn hidden_mail_folder_has_is_subscribed_zero() { server .mock( "GET", - "/me/mailFolders/FVISIBLE/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories", + "/me/mailFolders/FVISIBLE/messages?$top=100&$select=id,isRead,isDraft,flag,categories", ) .with_status(200) .with_header("content-type", "application/json") @@ -1405,7 +1426,7 @@ fn hidden_mail_folder_has_is_subscribed_zero() { server .mock( "GET", - "/me/mailFolders/FHIDDEN/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories", + "/me/mailFolders/FHIDDEN/messages?$top=100&$select=id,isRead,isDraft,flag,categories", ) .with_status(200) .with_header("content-type", "application/json") @@ -1483,7 +1504,7 @@ fn well_known_folder_probes_assign_jmap_roles() { server .mock( "GET", - format!("/me/mailFolders/{fid}/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories").as_str(), + format!("/me/mailFolders/{fid}/messages?$top=100&$select=id,isRead,isDraft,flag,categories").as_str(), ) .with_status(200) .with_header("content-type", "application/json") @@ -1557,7 +1578,10 @@ fn attachment_message_does_not_trigger_attachments_endpoint() { .create(); stub_well_known_folders(&mut server, "FMAIL"); server - .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"value":[{"id":"MSG-ATT"}]}"#) @@ -1754,7 +1778,7 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() { let _ids = server .mock( "GET", - "/me/mailFolders/AAkA-Padded%3D%3D/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories", + "/me/mailFolders/AAkA-Padded%3D%3D/messages?$top=100&$select=id,isRead,isDraft,flag,categories", ) .with_status(200) .with_header("content-type", "application/json") @@ -2162,7 +2186,10 @@ fn folder_enumeration_failure_skips_vanished_deletion() { .create(); } let _enum_fail = server - .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(503) .with_body("transient outage") .create(); @@ -2277,7 +2304,10 @@ fn stub_mail_surface(server: &mut Server) { .create(); stub_well_known_folders(server, "FMAIL"); server - .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories") + .mock( + "GET", + "/me/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"value":[{"id":"MSG-S"}]}"#) @@ -2652,7 +2682,7 @@ fn message_state_becomes_jmap_keywords() { } json_mock( &mut server, - "/me/mailFolders/F1/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories", + "/me/mailFolders/F1/messages?$top=100&$select=id,isRead,isDraft,flag,categories", r#"{"value":[ {"id":"M1","isRead":true,"isDraft":false,"isReadReceiptRequested":true, "flag":{"flagStatus":"flagged"},"categories":["Red Category","VIP"]}, @@ -2690,16 +2720,13 @@ fn message_state_becomes_jmap_keywords() { .map(|r| r.unwrap()) .collect(); let all = rows.join(" "); - for expected in [ - "$seen", - "$notified", - "$flagged", - "red category", - "vip", - "$draft", - ] { + for expected in ["$seen", "$flagged", "red_category", "vip", "$draft"] { assert!(all.contains(expected), "missing {expected} in {all}"); } + assert!( + !all.contains("$notified"), + "a read-receipt request is not a keyword: {all}" + ); } #[test] @@ -3095,7 +3122,7 @@ fn app_only_signs_in_as_the_app_and_reads_the_named_mailbox() { ), get( &mut server, - "/users/uid-1/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories", + "/users/uid-1/mailFolders/FMAIL/messages?$top=100&$select=id,isRead,isDraft,flag,categories", r#"{"value":[{"id":"MSG-1"}]}"#, ), ]; diff --git a/tests/sync_exchange_graph.rs b/tests/sync_exchange_graph.rs index f7d81ae..4063584 100644 --- a/tests/sync_exchange_graph.rs +++ b/tests/sync_exchange_graph.rs @@ -88,7 +88,7 @@ fn graph_fixture(server: &mut Server) { } json_mock( server, - "/me/mailFolders/F1/messages?$top=100&$select=id,isRead,isDraft,isReadReceiptRequested,flag,categories", + "/me/mailFolders/F1/messages?$top=100&$select=id,isRead,isDraft,flag,categories", r#"{"value":[{"id":"M1","isRead":true,"isDraft":false, "flag":{"flagStatus":"flagged"},"categories":["Work"]}]}"#, );