diff --git a/src/dav/parse.rs b/src/dav/parse.rs index a490574..6d9c006 100644 --- a/src/dav/parse.rs +++ b/src/dav/parse.rs @@ -7,7 +7,7 @@ use std::io::BufRead; use quick_xml::NsReader; -use quick_xml::events::Event; +use quick_xml::events::{BytesRef, Event}; use quick_xml::name::ResolveResult; use crate::dav::href::{Href, normalise}; @@ -116,10 +116,25 @@ enum Step { }, Text(String), CData(String), + Entity(char), Other, Eof, } +fn resolve_entity(g: &BytesRef) -> Option { + if let Ok(Some(c)) = g.resolve_char_ref() { + return Some(c); + } + match g.decode().ok()?.as_ref() { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + _ => None, + } +} + fn next_step(xml: &mut NsReader, buf: &mut Vec) -> Result { buf.clear(); let (ns, ev) = xml.read_resolved_event_into(buf)?; @@ -158,6 +173,10 @@ fn next_step(xml: &mut NsReader, buf: &mut Vec) -> Result Step::CData(String::from_utf8_lossy(cd.as_ref()).into_owned()), + Event::GeneralRef(g) => match resolve_entity(&g) { + Some(c) => Step::Entity(c), + None => Step::Other, + }, Event::Eof => Step::Eof, _ => Step::Other, }) @@ -445,6 +464,7 @@ fn read_text(xml: &mut NsReader) -> Result { let step = next_step(xml, &mut buf)?; match step { Step::Text(s) | Step::CData(s) => out.push_str(&s), + Step::Entity(c) => out.push(c), Step::StartElement { local, .. } => skip_element(xml, &local)?, Step::EndElement { .. } | Step::EmptyElement { .. } => break, Step::Eof => return Err(ParseError::Xml("unexpected EOF reading text".into())), diff --git a/src/exchange_ews/autodiscover.rs b/src/exchange_ews/autodiscover.rs index e3d6e72..4441829 100644 --- a/src/exchange_ews/autodiscover.rs +++ b/src/exchange_ews/autodiscover.rs @@ -12,6 +12,7 @@ use ureq::config::Config; use ureq::tls::{RootCerts, TlsConfig}; use crate::exchange_ews::error::EwsError; +use crate::exchange_ews::parse::entity_to_char; const V2_HOST: &str = "https://outlook.office365.com"; const POX_REQ_NS: &str = @@ -425,12 +426,14 @@ fn parse_pox_response(body: &[u8]) -> Result { let mut redirect_addr: Option = None; let mut redirect_url: Option = None; let mut action: Option = None; + let mut cur = String::new(); loop { buf.clear(); let (_, ev) = xml.read_resolved_event_into(&mut buf)?; match ev { Event::Start(e) => { let local = e.local_name().as_ref().to_vec(); + cur.clear(); if local.eq_ignore_ascii_case(b"Protocol") { current_type = None; } @@ -454,20 +457,31 @@ fn parse_pox_response(body: &[u8]) -> Result { } Event::End(e) => { let local = e.local_name().as_ref().to_vec(); + if let Some(field) = current.take() { + let text = std::mem::take(&mut cur); + match field { + "type" => current_type = Some(text), + "ewsUrl" => ews_url = Some(text), + "action" => action = Some(text), + "redirectAddr" => redirect_addr = Some(text), + "redirectUrl" => redirect_url = Some(text), + _ => {} + } + } + cur.clear(); if local.eq_ignore_ascii_case(b"Protocol") { current_type = None; } - current = None; } Event::Text(t) => { - let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); - match current { - Some("type") => current_type = Some(text), - Some("ewsUrl") => ews_url = Some(text), - Some("action") => action = Some(text), - Some("redirectAddr") => redirect_addr = Some(text), - Some("redirectUrl") => redirect_url = Some(text), - _ => {} + cur.push_str(&t.decode().map(|c| c.into_owned()).unwrap_or_default()); + } + Event::CData(c) => { + cur.push_str(&String::from_utf8_lossy(c.as_ref())); + } + Event::GeneralRef(g) => { + if let Some(c) = entity_to_char(&g) { + cur.push(c); } } Event::Eof => break, diff --git a/src/exchange_ews/parse.rs b/src/exchange_ews/parse.rs index 21e62e2..2eb6108 100644 --- a/src/exchange_ews/parse.rs +++ b/src/exchange_ews/parse.rs @@ -8,7 +8,7 @@ use std::io::BufRead; use quick_xml::NsReader; use quick_xml::XmlVersion; -use quick_xml::events::{BytesStart, Event}; +use quick_xml::events::{BytesRef, BytesStart, Event}; use quick_xml::name::ResolveResult; use crate::exchange_ews::error::EwsError; @@ -211,7 +211,7 @@ fn parse_fault(xml: &mut NsReader) -> Result let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); match text_target { Some("faultcode") => fault_code = text, - Some("faultstring") => fault_string = text, + Some("faultstring") => fault_string.push_str(&text), Some("responseCode") => code = ResponseCode::parse(&text), Some("messageXmlValue") if last_value_name.as_deref() == Some("BackOffMilliseconds") => @@ -222,7 +222,12 @@ fn parse_fault(xml: &mut NsReader) -> Result } } Event::CData(c) if text_target == Some("faultstring") => { - fault_string = String::from_utf8_lossy(c.as_ref()).into_owned(); + fault_string.push_str(&String::from_utf8_lossy(c.as_ref())); + } + Event::GeneralRef(ref g) if text_target == Some("faultstring") => { + if let Some(c) = entity_to_char(g) { + fault_string.push(c); + } } Event::Eof => break, _ => {} @@ -395,6 +400,7 @@ fn parse_folder_element( xml: &mut NsReader, element: FolderElement, ) -> Result { + xml.config_mut().trim_text(false); let mut entry = FolderEntry { element, ..FolderEntry::default() @@ -402,6 +408,7 @@ fn parse_folder_element( let mut buf = Vec::new(); let mut depth: u32 = 1; let mut current: Option<&'static str> = None; + let mut cur = String::new(); loop { buf.clear(); let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; @@ -429,12 +436,16 @@ fn parse_folder_element( } } else if local.eq_ignore_ascii_case(b"DisplayName") { current = Some("displayName"); + cur.clear(); } else if local.eq_ignore_ascii_case(b"FolderClass") { current = Some("folderClass"); + cur.clear(); } else if local.eq_ignore_ascii_case(b"TotalCount") { current = Some("totalCount"); + cur.clear(); } else if local.eq_ignore_ascii_case(b"ChildFolderCount") { current = Some("childCount"); + cur.clear(); } else { current = None; } @@ -444,23 +455,33 @@ fn parse_folder_element( } } Event::End(_) => { + if let Some(field) = current.take() { + let text = std::mem::take(&mut cur); + match field { + "displayName" => entry.display_name = text.trim().to_owned(), + "folderClass" => entry.folder_class = text.trim().to_owned(), + "totalCount" => entry.total_count = text.trim().parse().ok(), + "childCount" => entry.child_count = text.trim().parse().ok(), + _ => {} + } + } if depth == 0 { break; } depth -= 1; - current = None; if depth == 0 { break; } } Event::Text(t) => { - let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); - match current { - Some("displayName") => entry.display_name = text, - Some("folderClass") => entry.folder_class = text, - Some("totalCount") => entry.total_count = text.trim().parse().ok(), - Some("childCount") => entry.child_count = text.trim().parse().ok(), - _ => {} + cur.push_str(&t.decode().map(|c| c.into_owned()).unwrap_or_default()); + } + Event::CData(c) => { + cur.push_str(&String::from_utf8_lossy(c.as_ref())); + } + Event::GeneralRef(ref g) => { + if let Some(c) = entity_to_char(g) { + cur.push(c); } } Event::Eof => break, @@ -530,12 +551,27 @@ pub fn parse_find_item_response(body: &[u8]) -> Result Option { + if let Ok(Some(c)) = g.resolve_char_ref() { + return Some(c); + } + match g.decode().ok()?.as_ref() { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + _ => None, + } +} + fn is_item_element(local: &[u8]) -> bool { matches!( local.to_ascii_lowercase().as_slice(), b"message" | b"calendaritem" | b"contact" + | b"distributionlist" | b"meetingrequest" | b"meetingresponse" | b"meetingmessage" @@ -664,6 +700,15 @@ fn parse_one_response_message( write_text_xml(&mut inner, &text); } } + Event::GeneralRef(ref g) => { + if in_capture { + inner.push('&'); + if let Ok(name) = g.decode() { + inner.push_str(&name); + } + inner.push(';'); + } + } Event::Eof => break, _ => {} } @@ -911,6 +956,7 @@ pub fn parse_message_item(inner_xml: &str) -> Result { let mut mime_charset: Option = None; let mut category_collecting = false; let mut in_flag = false; + let mut cur = String::new(); loop { buf.clear(); let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; @@ -922,6 +968,7 @@ pub fn parse_message_item(inner_xml: &str) -> Result { if item.element.is_empty() && is_item_element(&local) { item.element = String::from_utf8_lossy(&local).into_owned(); } else if ns_kind == Ns::Types { + cur.clear(); if local.eq_ignore_ascii_case(b"ItemId") { capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); } else if local.eq_ignore_ascii_case(b"ParentFolderId") { @@ -963,41 +1010,42 @@ pub fn parse_message_item(inner_xml: &str) -> Result { Event::End(e) => { let local = e.local_name().as_ref().to_vec(); let lower = local.to_ascii_lowercase(); + if let Some(field) = current.take() { + let text = std::mem::take(&mut cur); + match field { + "mimeContent" => { + item.mime_content = Some(text); + item.mime_charset = mime_charset.clone(); + } + "subject" => item.subject = Some(text), + "received" => item.date_time_received = Some(text), + "isRead" => item.is_read = Some(matches!(text.trim(), "true" | "1")), + "isDraft" => item.is_draft = Some(matches!(text.trim(), "true" | "1")), + "readReceipt" => { + item.is_read_receipt_requested = + Some(matches!(text.trim(), "true" | "1")); + } + "category" => item.categories.push(text), + "flagStatus" => item.flag_status = Some(text), + _ => {} + } + } + cur.clear(); if lower == b"categories" { category_collecting = false; } else if lower == b"flag" { in_flag = false; } - current = None; } - Event::Text(t) => { - let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); - match current { - Some("mimeContent") => { - item.mime_content - .get_or_insert_with(String::new) - .push_str(&text); - item.mime_charset = mime_charset.clone(); - } - Some("subject") => item.subject = Some(text), - Some("received") => item.date_time_received = Some(text), - Some("isRead") => item.is_read = Some(matches!(text.trim(), "true" | "1")), - Some("isDraft") => item.is_draft = Some(matches!(text.trim(), "true" | "1")), - Some("readReceipt") => { - item.is_read_receipt_requested = Some(matches!(text.trim(), "true" | "1")); - } - Some("category") => item.categories.push(text), - Some("flagStatus") => item.flag_status = Some(text), - _ => {} - } + Event::Text(ref t) => { + cur.push_str(&t.decode().map(|c| c.into_owned()).unwrap_or_default()); } - Event::CData(c) => { - let text = String::from_utf8_lossy(c.as_ref()).into_owned(); - if current == Some("mimeContent") { - item.mime_content - .get_or_insert_with(String::new) - .push_str(&text); - item.mime_charset = mime_charset.clone(); + Event::CData(ref c) => { + cur.push_str(&String::from_utf8_lossy(c.as_ref())); + } + Event::GeneralRef(ref g) => { + if let Some(c) = entity_to_char(g) { + cur.push(c); } } Event::Eof => break, @@ -1139,6 +1187,7 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result let mut recurrence_text: Option<&'static str> = None; let mut recurrence = RawRecurrence::default(); let mut pending = PendingRecurrence::default(); + let mut cur = String::new(); loop { buf.clear(); @@ -1200,6 +1249,7 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result } continue; } + cur.clear(); if local.eq_ignore_ascii_case(b"ItemId") { if in_modified { if let Some(occ) = occurrence_stack.last_mut() { @@ -1350,6 +1400,80 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result recurrence_text = None; continue; } + if let Some(tt) = text_target { + let text = std::mem::take(&mut cur); + match tt { + "uid" => item.uid = Some(text), + "subject" => item.subject = Some(text), + "start" => item.start = Some(text), + "end" => item.end = Some(text), + "originalStart" => item.original_start = Some(text), + "isAllDay" => { + item.is_all_day_event = Some(matches!(text.trim(), "true" | "1")); + } + "freeBusy" => item.legacy_free_busy_status = Some(text), + "location" => item.location = Some(text), + "calendarItemType" => { + item.calendar_item_type = CalendarItemType::parse(text.trim()); + } + "recurrenceId" => item.recurrence_id = Some(text), + "organizerName" => item.organizer_name = Some(text), + "organizerEmail" => item.organizer_smtp = Some(text), + "attendeeName" => { + if let Some(att) = current_attendee.as_mut() { + att.name = Some(text); + } + } + "attendeeEmail" => { + if let Some(att) = current_attendee.as_mut() { + att.email = Some(text); + } + } + "attendeeResponse" => { + if let Some(att) = current_attendee.as_mut() { + att.response_type = Some(text); + } + } + "reminderIsSet" => { + item.reminder_is_set = Some(matches!(text.trim(), "true" | "1")); + } + "reminderMinutes" => { + item.reminder_minutes_before_start = text.trim().parse().ok(); + } + "isOnlineMeeting" => { + item.is_online_meeting = Some(matches!(text.trim(), "true" | "1")); + } + "joinUrl" => item.join_online_meeting_url = Some(text), + "netShowUrl" => item.net_show_url = Some(text), + "workspaceUrl" => item.meeting_workspace_url = Some(text), + "category" => item.categories.push(text), + "created" => item.created = Some(text), + "lastModified" => item.last_modified = Some(text), + "bodyText" => item.body_text = Some(text), + "bodyHtml" => item.body_html = Some(text), + "occStart" => { + if let Some(occ) = occurrence_stack.last_mut() { + occ.start = Some(text.clone()); + } + if let Some(occ) = deleted_stack.last_mut() { + occ.start = Some(text); + } + } + "occEnd" => { + if let Some(occ) = occurrence_stack.last_mut() { + occ.end = Some(text); + } + } + "occOrig" => { + if let Some(occ) = occurrence_stack.last_mut() { + occ.original_start = Some(text); + } + } + _ => {} + } + } + cur.clear(); + text_target = None; match lower.as_slice() { b"modifiedoccurrences" => in_modified = false, b"deletedoccurrences" => in_deleted = false, @@ -1381,14 +1505,9 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result b"categories" => category_collecting = false, _ => {} } - text_target = None; } - Event::Text(_) | Event::CData(_) => { - let text = match ev { - Event::Text(ref t) => t.decode().map(|c| c.into_owned()).unwrap_or_default(), - Event::CData(ref c) => String::from_utf8_lossy(c.as_ref()).into_owned(), - _ => unreachable!(), - }; + Event::Text(ref t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); if !recurrence_path.is_empty() { match recurrence_text { Some("interval") => pending.interval = text.trim().parse().unwrap_or(1), @@ -1411,86 +1530,20 @@ pub fn parse_calendar_item(inner_xml: &str) -> Result } _ => {} } - - continue; + } else { + cur.push_str(&text); } - - match text_target { - Some("uid") => item.uid = Some(text), - Some("subject") => item.subject = Some(text), - Some("start") => item.start = Some(text), - Some("end") => item.end = Some(text), - Some("originalStart") => item.original_start = Some(text), - Some("isAllDay") => { - item.is_all_day_event = Some(matches!(text.trim(), "true" | "1")); - } - Some("freeBusy") => item.legacy_free_busy_status = Some(text), - Some("location") => item.location = Some(text), - Some("calendarItemType") => { - item.calendar_item_type = CalendarItemType::parse(text.trim()); - } - Some("recurrenceId") => item.recurrence_id = Some(text), - Some("organizerName") => item.organizer_name = Some(text), - Some("organizerEmail") => item.organizer_smtp = Some(text), - Some("attendeeName") => { - if let Some(att) = current_attendee.as_mut() { - att.name = Some(text); - } - } - Some("attendeeEmail") => { - if let Some(att) = current_attendee.as_mut() { - att.email = Some(text); - } - } - Some("attendeeResponse") => { - if let Some(att) = current_attendee.as_mut() { - att.response_type = Some(text); - } - } - Some("reminderIsSet") => { - item.reminder_is_set = Some(matches!(text.trim(), "true" | "1")); - } - Some("reminderMinutes") => { - item.reminder_minutes_before_start = text.trim().parse().ok(); - } - Some("isOnlineMeeting") => { - item.is_online_meeting = Some(matches!(text.trim(), "true" | "1")); - } - Some("joinUrl") => item.join_online_meeting_url = Some(text), - Some("netShowUrl") => item.net_show_url = Some(text), - Some("workspaceUrl") => item.meeting_workspace_url = Some(text), - Some("category") => item.categories.push(text), - Some("created") => item.created = Some(text), - Some("lastModified") => item.last_modified = Some(text), - Some("bodyText") => { - item.body_text - .get_or_insert_with(String::new) - .push_str(&text); - } - Some("bodyHtml") => { - item.body_html - .get_or_insert_with(String::new) - .push_str(&text); - } - Some("occStart") => { - if let Some(occ) = occurrence_stack.last_mut() { - occ.start = Some(text.clone()); - } - if let Some(occ) = deleted_stack.last_mut() { - occ.start = Some(text); - } - } - Some("occEnd") => { - if let Some(occ) = occurrence_stack.last_mut() { - occ.end = Some(text); - } - } - Some("occOrig") => { - if let Some(occ) = occurrence_stack.last_mut() { - occ.original_start = Some(text); - } - } - _ => {} + } + Event::CData(ref c) => { + if recurrence_path.is_empty() { + cur.push_str(&String::from_utf8_lossy(c.as_ref())); + } + } + Event::GeneralRef(ref g) => { + if recurrence_path.is_empty() + && let Some(c) = entity_to_char(g) + { + cur.push(c); } } Event::Eof => break, @@ -1631,6 +1684,7 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { let mut in_members = false; let mut member_mailbox = false; let mut current_member: Option = None; + let mut cur = String::new(); loop { buf.clear(); let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; @@ -1652,6 +1706,7 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { if ns_kind != Ns::Types { continue; } + cur.clear(); if local.eq_ignore_ascii_case(b"ItemId") { capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); } else if local.eq_ignore_ascii_case(b"ParentFolderId") { @@ -1775,6 +1830,71 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { Event::End(e) => { let local = e.local_name().as_ref().to_vec(); let lower = local.to_ascii_lowercase(); + if let Some(at) = address_text { + if let Some(addr) = current_address.as_mut() { + let text = std::mem::take(&mut cur); + match at { + "street" => addr.street = Some(text), + "city" => addr.city = Some(text), + "state" => addr.state = Some(text), + "country" => addr.country = Some(text), + "postal" => addr.postal_code = Some(text), + _ => {} + } + } + } else if let Some(tt) = text_target { + let text = std::mem::take(&mut cur); + match tt { + "displayName" => item.display_name = Some(text), + "givenName" => item.given_name = Some(text), + "middleName" => item.middle_name = Some(text), + "surname" => item.surname = Some(text), + "initials" => item.initials = Some(text), + "nickname" => item.nickname = Some(text), + "companyName" => item.company_name = Some(text), + "department" => item.department = Some(text), + "jobTitle" => item.job_title = Some(text), + "generation" => item.generation = Some(text), + "officeLocation" => item.office_location = Some(text), + "url" => item.url = Some(text), + "birthday" => item.birthday = Some(text), + "weddingAnniversary" => item.wedding_anniversary = Some(text), + "manager" => item.manager = Some(text), + "spouse" => item.spouse = Some(text), + "assistant" => item.assistant = Some(text), + "profession" => item.profession = Some(text), + "postalAddressIndex" => item.postal_address_index = Some(text), + "notes" => item.notes = Some(text), + "created" => item.created = Some(text), + "lastModified" => item.last_modified = Some(text), + "memberName" => { + if let Some(m) = current_member.as_mut() { + m.name = Some(text); + } + } + "memberEmail" => { + if let Some(m) = current_member.as_mut() { + m.email = Some(text); + } + } + "category" => item.categories.push(text), + "child" => item.children.push(text), + "company" => item.companies.push(text), + "entryValue" => { + let key = entry_key.clone().unwrap_or_default(); + match entry_container { + Some("email") => item.emails.push((key, text)), + Some("phone") => item.phones.push((key, text)), + Some("im") => item.ims.push((key, text)), + _ => {} + } + } + _ => {} + } + } + cur.clear(); + text_target = None; + address_text = None; match lower.as_slice() { b"categories" => category_collecting = false, b"children" => children_collecting = false, @@ -1788,9 +1908,6 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { } entry_key = None; } - b"street" | b"city" | b"state" | b"countryorregion" | b"postalcode" => { - address_text = None; - } b"members" => in_members = false, b"mailbox" => member_mailbox = false, b"member" => { @@ -1802,74 +1919,16 @@ pub fn parse_contact_item(inner_xml: &str) -> Result { } _ => {} } - text_target = None; } - Event::Text(_) | Event::CData(_) => { - let text = match ev { - Event::Text(ref t) => t.decode().map(|c| c.into_owned()).unwrap_or_default(), - Event::CData(ref c) => String::from_utf8_lossy(c.as_ref()).into_owned(), - _ => unreachable!(), - }; - - if let Some(at) = address_text - && let Some(addr) = current_address.as_mut() - { - match at { - "street" => addr.street = Some(text.clone()), - "city" => addr.city = Some(text.clone()), - "state" => addr.state = Some(text.clone()), - "country" => addr.country = Some(text.clone()), - "postal" => addr.postal_code = Some(text.clone()), - _ => {} - } - continue; - } - match text_target { - Some("displayName") => item.display_name = Some(text), - Some("givenName") => item.given_name = Some(text), - Some("middleName") => item.middle_name = Some(text), - Some("surname") => item.surname = Some(text), - Some("initials") => item.initials = Some(text), - Some("nickname") => item.nickname = Some(text), - Some("companyName") => item.company_name = Some(text), - Some("department") => item.department = Some(text), - Some("jobTitle") => item.job_title = Some(text), - Some("generation") => item.generation = Some(text), - Some("officeLocation") => item.office_location = Some(text), - Some("url") => item.url = Some(text), - Some("birthday") => item.birthday = Some(text), - Some("weddingAnniversary") => item.wedding_anniversary = Some(text), - Some("manager") => item.manager = Some(text), - Some("spouse") => item.spouse = Some(text), - Some("assistant") => item.assistant = Some(text), - Some("profession") => item.profession = Some(text), - Some("postalAddressIndex") => item.postal_address_index = Some(text), - Some("notes") => item.notes = Some(text), - Some("created") => item.created = Some(text), - Some("lastModified") => item.last_modified = Some(text), - Some("memberName") => { - if let Some(m) = current_member.as_mut() { - m.name = Some(text); - } - } - Some("memberEmail") => { - if let Some(m) = current_member.as_mut() { - m.email = Some(text); - } - } - Some("category") => item.categories.push(text), - Some("child") => item.children.push(text), - Some("company") => item.companies.push(text), - Some("entryValue") => { - let key = entry_key.clone().unwrap_or_default(); - match entry_container { - Some("email") => item.emails.push((key, text)), - Some("phone") => item.phones.push((key, text)), - Some("im") => item.ims.push((key, text)), - _ => {} - } - } - _ => {} + Event::Text(ref t) => { + cur.push_str(&t.decode().map(|c| c.into_owned()).unwrap_or_default()); + } + Event::CData(ref c) => { + cur.push_str(&String::from_utf8_lossy(c.as_ref())); + } + Event::GeneralRef(ref g) => { + if let Some(c) = entity_to_char(g) { + cur.push(c); } } Event::Eof => break, @@ -1890,6 +1949,7 @@ fn parse_attachment_ref( let mut buf = Vec::new(); let mut depth: u32 = 1; let mut current: Option<&'static str> = None; + let mut cur = String::new(); loop { buf.clear(); let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; @@ -1902,6 +1962,7 @@ fn parse_attachment_ref( } let local = e.local_name().as_ref().to_vec(); if ns_kind == Ns::Types { + cur.clear(); if local.eq_ignore_ascii_case(b"AttachmentId") { if let Some(v) = attr_value(e, b"Id") { att.attachment_id = v; @@ -1921,24 +1982,35 @@ fn parse_attachment_ref( } } Event::End(_) => { + if let Some(field) = current.take() { + let text = std::mem::take(&mut cur); + match field { + "name" => att.name = Some(text), + "contentType" => att.content_type = Some(text), + "isContactPhoto" => { + att.is_contact_photo = matches!(text.trim(), "true" | "1"); + } + _ => {} + } + } + cur.clear(); if depth == 0 { break; } depth -= 1; - current = None; if depth == 0 { break; } } - Event::Text(t) => { - let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); - match current { - Some("name") => att.name = Some(text), - Some("contentType") => att.content_type = Some(text), - Some("isContactPhoto") => { - att.is_contact_photo = matches!(text.trim(), "true" | "1"); - } - _ => {} + Event::Text(ref t) => { + cur.push_str(&t.decode().map(|c| c.into_owned()).unwrap_or_default()); + } + Event::CData(ref c) => { + cur.push_str(&String::from_utf8_lossy(c.as_ref())); + } + Event::GeneralRef(ref g) => { + if let Some(c) = entity_to_char(g) { + cur.push(c); } } Event::Eof => break, @@ -2019,8 +2091,14 @@ pub fn parse_get_attachment_inline(body: &[u8]) -> Result cur.name = Some(text), - Some("contentType") => cur.content_type = Some(text), + Some("name") => { + cur.name.get_or_insert_with(String::new).push_str(&text); + } + Some("contentType") => { + cur.content_type + .get_or_insert_with(String::new) + .push_str(&text); + } Some("isContactPhoto") => { cur.is_contact_photo = matches!(text.trim(), "true" | "1"); } @@ -2029,6 +2107,18 @@ pub fn parse_get_attachment_inline(body: &[u8]) -> Result { + if let (Some(cur), Some(c)) = (current.as_mut(), entity_to_char(g)) { + match text_target { + Some("name") => cur.name.get_or_insert_with(String::new).push(c), + Some("contentType") => { + cur.content_type.get_or_insert_with(String::new).push(c) + } + Some("content") => cur.content_base64.push(c), + _ => {} + } + } + } Event::Eof => break, _ => {} } @@ -2297,6 +2387,74 @@ mod tests { assert_eq!(parsed.members[1].name.as_deref(), Some("Carol")); } + #[test] + fn contact_decodes_xml_entities() { + let body = format!( + "\ + \ + R&D\ + Sales & Marketing <Team>\ + AT&T\ + R&D\ + a&b@x.com\ + " + ); + let parsed = parse_contact_item(&body).unwrap(); + assert_eq!( + parsed.display_name.as_deref(), + Some("Sales & Marketing ") + ); + assert_eq!(parsed.company_name.as_deref(), Some("AT&T")); + assert_eq!(parsed.department.as_deref(), Some("R&D")); + assert_eq!(parsed.categories, vec!["R&D".to_owned()]); + assert_eq!(parsed.emails[0].1, "a&b@x.com"); + } + + #[test] + fn calendar_decodes_xml_entities_in_subject_and_location() { + let body = format!( + "\ + \ + Q&A <all hands>\ + Room A & B\ + 2025-06-15T14:00:00Z\ + 2025-06-15T15:00:00Z\ + " + ); + let parsed = parse_calendar_item(&body).unwrap(); + assert_eq!(parsed.subject.as_deref(), Some("Q&A ")); + assert_eq!(parsed.location.as_deref(), Some("Room A & B")); + } + + #[test] + fn folder_decodes_xml_entities_in_display_name() { + let body = format!( + "\ + Sales & Marketing\ + IPF.Note" + ); + let entry = parse_folder_inner(&body).unwrap().unwrap(); + assert_eq!(entry.display_name, "Sales & Marketing"); + } + + #[test] + fn response_capture_preserves_entities_for_reparse() { + let body = format!( + "\ + NoError\ + AT&T\ + " + ); + let msgs = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + assert!( + msgs[0].inner_xml.contains("AT&T") || msgs[0].inner_xml.contains("AT&T"), + "capture must preserve the entity for the per-item parser: {}", + msgs[0].inner_xml + ); + let parsed = parse_contact_item(&msgs[0].inner_xml).unwrap(); + assert_eq!(parsed.company_name.as_deref(), Some("AT&T")); + } + #[test] fn contact_parses_emails_phones_addresses() { let body = format!( diff --git a/src/exchange_ews/xml.rs b/src/exchange_ews/xml.rs index 45adad9..d89b077 100644 --- a/src/exchange_ews/xml.rs +++ b/src/exchange_ews/xml.rs @@ -339,8 +339,11 @@ mod tests { #[test] fn get_item_calendar_shape_lists_calendar_fields() { - let body = - get_item_body(ItemShape::CalendarItem, &[ItemId::new("X", "")], ServerVersion::Exchange2010Sp2); + let body = get_item_body( + ItemShape::CalendarItem, + &[ItemId::new("X", "")], + ServerVersion::Exchange2010Sp2, + ); assert!(body.contains("calendar:Recurrence")); assert!(body.contains("calendar:ModifiedOccurrences")); assert!(body.contains("calendar:DeletedOccurrences")); @@ -350,8 +353,11 @@ mod tests { #[test] fn get_item_contact_shape_requests_all_properties() { - let body = - get_item_body(ItemShape::Contact, &[ItemId::new("X", "")], ServerVersion::Exchange2013Sp1); + let body = get_item_body( + ItemShape::Contact, + &[ItemId::new("X", "")], + ServerVersion::Exchange2013Sp1, + ); assert!(body.contains("AllProperties")); } @@ -379,12 +385,18 @@ mod tests { #[test] fn calendar_timezone_fields_gated_on_exchange_2010() { - let modern = - get_item_body(ItemShape::CalendarItem, &[ItemId::new("X", "")], ServerVersion::Exchange2010); + let modern = get_item_body( + ItemShape::CalendarItem, + &[ItemId::new("X", "")], + ServerVersion::Exchange2010, + ); assert!(modern.contains("calendar:StartTimeZone")); assert!(modern.contains("calendar:EndTimeZone")); - let legacy = - get_item_body(ItemShape::CalendarItem, &[ItemId::new("X", "")], ServerVersion::Exchange2007); + let legacy = get_item_body( + ItemShape::CalendarItem, + &[ItemId::new("X", "")], + ServerVersion::Exchange2007, + ); assert!( !legacy.contains("TimeZone"), "StartTimeZone/EndTimeZone are Exchange 2010 additions" diff --git a/src/sync/import_exchange_ews/calendar.rs b/src/sync/import_exchange_ews/calendar.rs index 819abd0..46ea18c 100644 --- a/src/sync/import_exchange_ews/calendar.rs +++ b/src/sync/import_exchange_ews/calendar.rs @@ -19,7 +19,7 @@ use crate::sync::TypeCounts; use super::attachments::{fetch_attachments, intern_attachment}; use super::folders::FolderPlan; use super::items::{ - EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for, + EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for, }; pub fn reconcile_all( @@ -41,9 +41,18 @@ pub fn reconcile_all( Some(id) => id, None => continue, }; - if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) { - ctx.logger - .warn(&format!("calendar folder {} failed: {}", folder_id.id, e)); + if let Err(e) = reconcile_one( + conn, + ctx, + folder_id, + &folder.folder.display_name, + local_folder_id, + counts, + ) { + ctx.logger.warn(&format!( + "calendar folder {:?} failed: {}", + folder.folder.display_name, e + )); counts.failed += 1; } } @@ -54,6 +63,7 @@ fn reconcile_one( conn: &mut Connection, ctx: &ItemRunCtx<'_>, folder: &crate::exchange_ews::types::FolderId, + folder_name: &str, local_folder_id: i64, counts: &mut TypeCounts, ) -> Result<(), Error> { @@ -81,8 +91,8 @@ fn reconcile_one( let plan = plan_for(&outcome, &local); if ctx.logger.enabled(LEVEL_PROGRESS) { eprintln!( - "EWS calendar folder {}: new={} changed={} vanished={}", - folder.id, + "EWS calendar folder {:?}: new={} changed={} vanished={}", + folder_name, plan.new.len(), plan.present_changed.len(), plan.vanished.len() @@ -93,9 +103,7 @@ fn reconcile_one( to_fetch.push(id.clone()); } if !to_fetch.is_empty() { - let outcome = get_items(ctx, ItemShape::CalendarItem, &to_fetch).map_err(Error::from)?; - counts.failed += outcome.failed_items; - for msg in outcome.messages { + let failed_items = for_each_fetched_item(ctx, ItemShape::CalendarItem, &to_fetch, |msg| { if !msg.success { if matches!( msg.response_code, @@ -107,19 +115,19 @@ fn reconcile_one( ctx.logger .warn(&format!("GetItem (calendar) error: {}", msg.response_code)); } - continue; + return Ok(()); } let parsed = parse_calendar_item(&msg.inner_xml).map_err(Error::from)?; if parsed.id.id.is_empty() { counts.failed += 1; - continue; + return Ok(()); } if matches!( parsed.calendar_item_type, Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception) ) { counts.skipped += 1; - continue; + return Ok(()); } let existing = plan .present_changed @@ -134,8 +142,9 @@ fn reconcile_one( &folder.id, existing, counts, - )?; - } + ) + })?; + counts.failed += failed_items; } delete_vanished( conn, diff --git a/src/sync/import_exchange_ews/contacts.rs b/src/sync/import_exchange_ews/contacts.rs index 2415948..07ebce5 100644 --- a/src/sync/import_exchange_ews/contacts.rs +++ b/src/sync/import_exchange_ews/contacts.rs @@ -18,7 +18,7 @@ use crate::sync::TypeCounts; use super::attachments::fetch_contact_photo; use super::folders::FolderPlan; use super::items::{ - EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for, + EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for, }; pub fn reconcile_all( @@ -40,9 +40,18 @@ pub fn reconcile_all( Some(id) => id, None => continue, }; - if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) { - ctx.logger - .warn(&format!("contact folder {} failed: {}", folder_id.id, e)); + if let Err(e) = reconcile_one( + conn, + ctx, + folder_id, + &folder.folder.display_name, + local_folder_id, + counts, + ) { + ctx.logger.warn(&format!( + "contact folder {:?} failed: {}", + folder.folder.display_name, e + )); counts.failed += 1; } } @@ -53,6 +62,7 @@ fn reconcile_one( conn: &mut Connection, ctx: &ItemRunCtx<'_>, folder: &crate::exchange_ews::types::FolderId, + folder_name: &str, local_folder_id: i64, counts: &mut TypeCounts, ) -> Result<(), Error> { @@ -74,8 +84,8 @@ fn reconcile_one( let plan = plan_for(&outcome, &local); if ctx.logger.enabled(LEVEL_PROGRESS) { eprintln!( - "EWS contacts folder {}: new={} changed={} vanished={}", - folder.id, + "EWS contacts folder {:?}: new={} changed={} vanished={}", + folder_name, plan.new.len(), plan.present_changed.len(), plan.vanished.len() @@ -86,9 +96,7 @@ fn reconcile_one( to_fetch.push(id.clone()); } if !to_fetch.is_empty() { - let outcome = get_items(ctx, ItemShape::Contact, &to_fetch).map_err(Error::from)?; - counts.failed += outcome.failed_items; - for msg in outcome.messages { + let failed_items = for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, |msg| { if !msg.success { if matches!( msg.response_code, @@ -100,12 +108,12 @@ fn reconcile_one( ctx.logger .warn(&format!("GetItem (contact) error: {}", msg.response_code)); } - continue; + return Ok(()); } let parsed = parse_contact_item(&msg.inner_xml).map_err(Error::from)?; if parsed.id.id.is_empty() { counts.failed += 1; - continue; + return Ok(()); } let existing = plan .present_changed @@ -120,8 +128,9 @@ fn reconcile_one( &folder.id, existing, counts, - )?; - } + ) + })?; + counts.failed += failed_items; } delete_vanished( conn, diff --git a/src/sync/import_exchange_ews/items.rs b/src/sync/import_exchange_ews/items.rs index 2accbf9..fec72f5 100644 --- a/src/sync/import_exchange_ews/items.rs +++ b/src/sync/import_exchange_ews/items.rs @@ -368,6 +368,29 @@ pub fn get_items( }) } +pub fn for_each_fetched_item( + ctx: &ItemRunCtx<'_>, + shape: ItemShape, + ids: &[ItemId], + mut on_message: F, +) -> Result +where + F: FnMut(crate::exchange_ews::parse::ResponseMessage) -> Result<(), Error>, +{ + let batch = ctx.batch_size.max(1); + let workers = ctx.connections.clamp(1, 8); + let window = batch.saturating_mul(workers).max(batch); + let mut failed_items = 0u64; + for win in ids.chunks(window) { + let outcome = get_items(ctx, shape, win).map_err(Error::from)?; + failed_items = failed_items.saturating_add(outcome.failed_items); + for msg in outcome.messages { + on_message(msg)?; + } + } + Ok(failed_items) +} + pub fn delete_vanished( conn: &mut rusqlite::Connection, source_id: i64, diff --git a/src/sync/import_exchange_ews/messages.rs b/src/sync/import_exchange_ews/messages.rs index 247c2be..1c77ba4 100644 --- a/src/sync/import_exchange_ews/messages.rs +++ b/src/sync/import_exchange_ews/messages.rs @@ -21,7 +21,7 @@ use crate::sync::keys::index_to_json; use super::folders::FolderPlan; use super::items::{ - EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for, + EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for, }; pub fn reconcile_all( @@ -43,9 +43,18 @@ pub fn reconcile_all( Some(id) => id, None => continue, }; - if let Err(e) = reconcile_one_folder(conn, ctx, folder_id, local_folder_id, counts) { - ctx.logger - .warn(&format!("email folder {} failed: {}", folder_id.id, e)); + if let Err(e) = reconcile_one_folder( + conn, + ctx, + folder_id, + &folder.folder.display_name, + local_folder_id, + counts, + ) { + ctx.logger.warn(&format!( + "email folder {:?} failed: {}", + folder.folder.display_name, e + )); counts.failed += 1; } } @@ -56,6 +65,7 @@ fn reconcile_one_folder( conn: &mut Connection, ctx: &ItemRunCtx<'_>, folder: &crate::exchange_ews::types::FolderId, + folder_name: &str, local_folder_id: i64, counts: &mut TypeCounts, ) -> Result<(), Error> { @@ -73,8 +83,8 @@ fn reconcile_one_folder( let plan = plan_for(&outcome, &local); if ctx.logger.enabled(LEVEL_PROGRESS) { eprintln!( - "EWS folder {}: new={} changed={} vanished={} unchanged={}", - folder.id, + "EWS folder {:?}: new={} changed={} vanished={} unchanged={}", + folder_name, plan.new.len(), plan.present_changed.len(), plan.vanished.len(), @@ -86,9 +96,7 @@ fn reconcile_one_folder( to_fetch.push(id.clone()); } if !to_fetch.is_empty() { - let outcome = get_items(ctx, ItemShape::Message, &to_fetch).map_err(Error::from)?; - counts.failed += outcome.failed_items; - for msg in outcome.messages { + let failed_items = for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, |msg| { if !msg.success { if matches!( msg.response_code, @@ -102,12 +110,12 @@ fn reconcile_one_folder( msg.response_code, msg.message_text )); } - continue; + return Ok(()); } let parsed = parse_message_item(&msg.inner_xml).map_err(Error::from)?; if parsed.id.id.is_empty() { counts.failed += 1; - continue; + return Ok(()); } let existing = plan .present_changed @@ -122,8 +130,9 @@ fn reconcile_one_folder( &folder.id, existing, counts, - )?; - } + ) + })?; + counts.failed += failed_items; } delete_vanished( conn, diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs index 6a39eca..cf121ed 100644 --- a/src/sync/import_jmap.rs +++ b/src/sync/import_jmap.rs @@ -430,13 +430,6 @@ fn insert_objects( logger: &Logger, counts: &mut TypeCounts, ) -> Result<(), Error> { - let blob_refs = blob_references(ty, &objects); - let blobs = if blob_refs.is_empty() { - HashMap::new() - } else { - download_blobs(net, blob_refs, threads, logger, counts) - }; - let existing_parents: HashSet = if matches!(ty, ObjectType::Mailbox | ObjectType::FileNode) { db::ids::jmap_to_local(&ctx.conn, source_id, ty) @@ -451,6 +444,12 @@ fn insert_objects( } for batch in ordered.chunks(200) { + let blob_refs = blob_references(ty, batch.iter().map(|&i| &objects[i])); + let blobs = if blob_refs.is_empty() { + HashMap::new() + } else { + download_blobs(net, blob_refs, threads, logger, counts) + }; let tx = ctx .conn .unchecked_transaction() @@ -481,7 +480,7 @@ fn insert_objects( Ok(()) } -fn blob_references(ty: ObjectType, objects: &[Value]) -> Vec { +fn blob_references<'a>(ty: ObjectType, objects: impl Iterator) -> Vec { let mut seen: HashSet = HashSet::new(); let mut refs = Vec::new(); match ty { @@ -845,14 +844,13 @@ fn update_objects( logger: &Logger, counts: &mut TypeCounts, ) -> Result<(), Error> { - let blob_refs = blob_references(ty, &objects); - let blobs = if blob_refs.is_empty() { - HashMap::new() - } else { - download_blobs(net, blob_refs, threads, logger, counts) - }; - for batch in objects.chunks(200) { + let blob_refs = blob_references(ty, batch.iter()); + let blobs = if blob_refs.is_empty() { + HashMap::new() + } else { + download_blobs(net, blob_refs, threads, logger, counts) + }; let tx = ctx .conn .unchecked_transaction() diff --git a/tests/mock_dav.rs b/tests/mock_dav.rs index f2cfe94..269d161 100644 --- a/tests/mock_dav.rs +++ b/tests/mock_dav.rs @@ -332,6 +332,33 @@ fn http_503_retries_then_succeeds() { assert!(c.retries_observed() >= 1); } +#[test] +fn xml_entities_decoded_in_displayname_and_calendar_data() { + let body = r#" + + + /cal/1.ics + + Sales & Marketing <Team> + BEGIN:VEVENT +SUMMARY:Q&A & review +END:VEVENT + HTTP/1.1 200 OK + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!(r.len(), 1); + assert_eq!( + r[0].props.displayname.as_deref(), + Some("Sales & Marketing ") + ); + let cal = r[0].props.calendar_data.as_deref().unwrap(); + assert!( + cal.contains("SUMMARY:Q&A & review"), + "calendar-data entities must be decoded: {cal}" + ); +} + #[test] fn duplicate_response_hrefs_collapsed_on_parse() { let body = r#" diff --git a/tests/mock_exchange_ews.rs b/tests/mock_exchange_ews.rs index 47c9318..580bb69 100644 --- a/tests/mock_exchange_ews.rs +++ b/tests/mock_exchange_ews.rs @@ -86,6 +86,31 @@ fn find_folder_pagination_and_classification() { assert_eq!(inbox.folder_class, "IPF.Note"); } +#[test] +fn find_item_distribution_list_does_not_clobber_preceding_contact() { + let body = envelope(&format!( + "\ + NoError\ + \ + \ + \ + \ + " + )); + let parsed = parse_find_item_response(body.as_bytes()).unwrap(); + assert_eq!( + parsed.items.len(), + 3, + "DistributionList must be its own item, not clobber C2" + ); + let ids: Vec<&str> = parsed.items.iter().map(|i| i.id.id.as_str()).collect(); + assert_eq!(ids, ["C1", "C2", "DL1"]); + assert_eq!( + parsed.items[2].element.to_ascii_lowercase(), + "distributionlist" + ); +} + #[test] fn find_item_offset_loop_terminates_on_includes_last_true() { let mut server = mockito::Server::new(); @@ -699,6 +724,53 @@ fn http_456_surfaces_as_account_locked_auth_error() { } } +#[test] +fn for_each_fetched_item_streams_every_id_across_windows() { + use vandelay::logging::Logger; + use vandelay::sync::import_exchange_ews::items::{ItemRunCtx, for_each_fetched_item}; + + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let one_message = envelope(&format!( + "\ + NoError\ + " + )); + let _m = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(&one_message) + .expect(5) + .create(); + + let c = client(0); + let ctx = ItemRunCtx { + client: &c, + url: &url, + source_id: 1, + batch_size: 1, + attachment_batch: 1, + connections: 2, + use_syncfolderitems: false, + sync_batch: 512, + logger: Logger::new(0), + }; + let ids: Vec = (0..5).map(|i| ItemId::new(format!("I{i}"), "K")).collect(); + + let mut delivered = 0usize; + let failed = for_each_fetched_item(&ctx, ItemShape::Message, &ids, |msg| { + assert!(msg.success); + delivered += 1; + Ok(()) + }) + .expect("streaming fetch should succeed"); + + assert_eq!(delivered, 5, "every id must be delivered exactly once"); + assert_eq!(failed, 0); + _m.assert(); +} + #[test] fn warning_response_class_is_treated_as_success_in_mock() { let body = envelope(&format!(