diff --git a/CHANGELOG.md b/CHANGELOG.md index 85d93f6..412684b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Added +- IMAP: `--mime-keywords` adds the phrases of each message's `Keywords:` + header as keywords. Exchange shows its categories over IMAP this way. + Each phrase is lowercased and its spaces become `_`. Letters outside ASCII + are dropped, and a phrase can't name a system keyword such as `$seen`. + +### Changed +- Upstream v1.0.15 is merged in. + +### Fixed +- Maildir: folder names kept in modified UTF-7 are now decoded in full + (`.&ZeVnLIqe-.&U,BTFw-` is `日本語.台北`). Folder selection and role + guessing use the decoded name. Before, only the last part of a name was + decoded, for display. +- Exchange EWS: with Basic or delegated OAuth sign-in, `--mailbox` naming + another mailbox the user has full access to imported the user's own + mailbox instead. Folders are now addressed in the mailbox `--mailbox` + names. + ## [27.1-rc.11] The eleventh release candidate of the 27.1 series: upstream v1.0.14, and diff --git a/src/cli.rs b/src/cli.rs index 151a34c..50211f8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -346,6 +346,12 @@ struct ImapImportArgs { #[arg(long, help = "Import \\Deleted messages with the $deleted keyword")] include_deleted: bool, + #[arg( + long, + help = "Add the phrases of each message's Keywords: header (Exchange categories) as JMAP keywords" + )] + mime_keywords: bool, + #[arg( long, value_name = "N", @@ -1391,6 +1397,7 @@ fn resolve_imap_import(args: ImapImportArgs) -> Result { subscribed_only: args.subscribed_only, automap: !args.noautomap, include_deleted: args.include_deleted, + mime_keywords: args.mime_keywords, fetch_batch: args.fetch_batch, fetch_batch_bytes: args.fetch_batch_mib.max(1).saturating_mul(1024 * 1024), imap_connections, diff --git a/src/exchange_ews/xml.rs b/src/exchange_ews/xml.rs index a12d284..979dfa7 100644 --- a/src/exchange_ews/xml.rs +++ b/src/exchange_ews/xml.rs @@ -11,9 +11,19 @@ use crate::exchange_ews::types::{DistinguishedFolderId, FolderId, ItemId, Server #[derive(Debug, Clone, Copy)] pub enum FolderRef<'a> { Distinguished(DistinguishedFolderId), + DistinguishedIn(DistinguishedFolderId, &'a str), Concrete(&'a FolderId), } +impl<'a> FolderRef<'a> { + pub fn distinguished(id: DistinguishedFolderId, mailbox: Option<&'a str>) -> FolderRef<'a> { + match mailbox { + Some(mailbox) => FolderRef::DistinguishedIn(id, mailbox), + None => FolderRef::Distinguished(id), + } + } +} + #[derive(Debug, Clone, Copy)] pub enum FolderShape { IdOnly, @@ -266,6 +276,13 @@ fn write_folder_ref(out: &mut String, parent: FolderRef<'_>) { out.push_str(d.as_str()); out.push_str("\"/>"); } + FolderRef::DistinguishedIn(d, mailbox) => { + out.push_str(""); + write_escaped(out, mailbox); + out.push_str(""); + } FolderRef::Concrete(f) => write_folder_id_element(out, f), } } @@ -305,6 +322,27 @@ mod tests { )); } + #[test] + fn distinguished_folder_in_named_mailbox_carries_escaped_address() { + let body = find_folder_body( + FolderRef::distinguished( + DistinguishedFolderId::MsgFolderRoot, + Some("b&o@corp.example"), + ), + Traversal::Deep, + ); + assert!(body.contains( + "\ + b&o@corp.example\ + " + )); + let body = find_folder_body( + FolderRef::distinguished(DistinguishedFolderId::MsgFolderRoot, None), + Traversal::Deep, + ); + assert!(body.contains("")); + } + #[test] fn find_item_paginates_with_offset_and_page_size() { let folder = FolderId::new("FID", "FCK"); diff --git a/src/sync/emailmeta.rs b/src/sync/emailmeta.rs index 104e1d4..df991bf 100644 --- a/src/sync/emailmeta.rs +++ b/src/sync/emailmeta.rs @@ -4,7 +4,9 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use mail_parser::{Address, HeaderName, HeaderValue, MessageParser}; +use std::slice; + +use mail_parser::{Address, HeaderName, HeaderValue, Message, MessageParser}; use crate::sync::keys::{EmailIndex, email_index}; @@ -30,10 +32,78 @@ pub fn email_index_from_blob(bytes: &[u8]) -> EmailIndex { email_meta_from_blob(bytes).0 } -pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option) { +const MAX_KEYWORD_LEN: usize = 255; +const MAX_MIME_KEYWORDS: usize = 64; + +pub fn email_index_and_mime_keywords(bytes: &[u8], keywords: &mut Vec) -> EmailIndex { let Some(m) = MessageParser::default().parse(bytes) else { - return (email_index(&[], &[], "", "", &[]), None); + return empty_meta().0; }; + add_keywords_header(&m, keywords); + email_meta(&m).0 +} + +pub fn add_mime_keywords(bytes: &[u8], keywords: &mut Vec) { + if let Some(m) = MessageParser::default().parse_headers(bytes) { + add_keywords_header(&m, keywords); + } +} + +fn add_keywords_header(m: &Message<'_>, keywords: &mut Vec) { + let mut added = 0; + for value in m.header_values(HeaderName::Keywords) { + let phrases = match value { + HeaderValue::Text(phrase) => slice::from_ref(phrase), + HeaderValue::TextList(phrases) => phrases.as_slice(), + _ => &[], + }; + for keyword in phrases.iter().filter_map(|phrase| jmap_keyword(phrase)) { + if !keywords.iter().any(|k| k.eq_ignore_ascii_case(&keyword)) { + keywords.push(keyword); + added += 1; + if added == MAX_MIME_KEYWORDS { + return; + } + } + } + } +} + +fn jmap_keyword(phrase: &str) -> Option { + let mut keyword = String::with_capacity(phrase.len().min(MAX_KEYWORD_LEN)); + let mut separator = false; + for ch in phrase.chars() { + if ch.is_ascii_graphic() + && !matches!(ch, '(' | ')' | '{' | ']' | '%' | '*' | '"' | '\\') + && !(ch == '$' && keyword.is_empty()) + { + let sep = separator && !keyword.is_empty(); + if keyword.len() + usize::from(sep) >= MAX_KEYWORD_LEN { + break; + } + if sep { + keyword.push('_'); + } + keyword.push(ch.to_ascii_lowercase()); + separator = false; + } else { + separator = true; + } + } + (!keyword.is_empty()).then_some(keyword) +} + +fn empty_meta() -> (EmailIndex, Option) { + (email_index(&[], &[], "", "", &[]), None) +} + +pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option) { + MessageParser::default() + .parse(bytes) + .map_or_else(empty_meta, |m| email_meta(&m)) +} + +fn email_meta(m: &Message<'_>) -> (EmailIndex, Option) { let mut mids: Vec = Vec::new(); for v in m.header_values(HeaderName::MessageId) { mids.extend(message_ids(v)); @@ -59,6 +129,77 @@ mod tests { use super::*; use crate::sync::keys::{EmailKey, email_key}; + // Every Keywords header and every comma-separated phrase is mapped to a JMAP keyword. + #[test] + fn mime_keywords_are_read_from_every_keywords_header() { + let mut keywords = vec!["$seen".to_owned(), "VIP".to_owned()]; + let raw = b"Message-ID: \r\nKeywords: bearbeitet von KL, Wiedervorlage\r\nKeywords: vip, =?UTF-8?Q?Pr=C3=BCfung?=\r\n\r\nbody"; + let idx = email_index_and_mime_keywords(raw, &mut keywords); + assert_eq!(idx.mids, vec!["k@h".to_owned()]); + assert_eq!( + keywords, + vec![ + "$seen", + "VIP", + "bearbeitet_von_kl", + "wiedervorlage", + "pr_fung" + ] + ); + + let mut from_headers = vec!["$seen".to_owned(), "VIP".to_owned()]; + add_mime_keywords(raw, &mut from_headers); + assert_eq!(from_headers, keywords); + } + + // Phrases are reduced to the RFC 8621 keyword alphabet, without a leading "$", within 255 chars. + #[test] + fn jmap_keyword_follows_rfc8621_syntax() { + assert_eq!( + jmap_keyword(" Review / Legal ").as_deref(), + Some("review_/_legal") + ); + assert_eq!( + jmap_keyword("a(b)c{d]e%f*g\"h\\i").as_deref(), + Some("a_b_c_d_e_f_g_h_i") + ); + assert_eq!(jmap_keyword("$seen").as_deref(), Some("seen")); + assert_eq!(jmap_keyword(" $$Junk").as_deref(), Some("junk")); + assert_eq!(jmap_keyword("a$b").as_deref(), Some("a$b")); + assert_eq!( + jmap_keyword("Q4-2026.final_v2").as_deref(), + Some("q4-2026.final_v2") + ); + assert_eq!(jmap_keyword(" ( ) "), None); + assert_eq!( + jmap_keyword("x".repeat(300).as_str()).map(|k| k.len()), + Some(255) + ); + assert_eq!( + jmap_keyword(&format!("{} b", "a".repeat(254))).map(|k| k.len()), + Some(254) + ); + } + + // Empty phrases, messages without the header and the keyword count cap. + #[test] + fn mime_keywords_skip_empty_and_are_capped() { + let mut keywords = Vec::new(); + add_mime_keywords(b"Keywords: , \r\n\r\nbody", &mut keywords); + add_mime_keywords(b"Subject: none\r\n\r\nbody", &mut keywords); + assert!(keywords.is_empty()); + + let many = (0..100) + .map(|n| format!("k{n}")) + .collect::>() + .join(", "); + add_mime_keywords( + format!("Keywords: {many}\r\n\r\nbody").as_bytes(), + &mut keywords, + ); + assert_eq!(keywords.len(), MAX_MIME_KEYWORDS); + } + #[test] fn parses_core_fields_and_keys_by_message_id() { let raw = b"From: Alice \r\nTo: Bob \r\nSubject: Hi there\r\nMessage-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody\r\n"; diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs index 51328c4..eeabf1d 100644 --- a/src/sync/import_exchange_ews/coordinator.rs +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -117,8 +117,14 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result Result { - let plan = - plan_folders(client, session_url, config.mailbox_kind, logger).map_err(Error::from)?; + let plan = plan_folders( + client, + session_url, + config.mailbox_kind, + config.mailbox.as_deref(), + logger, + ) + .map_err(Error::from)?; let mut summary = Summary::default(); let mut mailbox_counts = TypeCounts::default(); let mut calendar_counts = TypeCounts::default(); diff --git a/src/sync/import_exchange_ews/folders.rs b/src/sync/import_exchange_ews/folders.rs index fc4a791..0c25831 100644 --- a/src/sync/import_exchange_ews/folders.rs +++ b/src/sync/import_exchange_ews/folders.rs @@ -41,10 +41,12 @@ pub fn plan_folders( client: &EwsClient, url: &str, mailbox_kind: MailboxKind, + mailbox: Option<&str>, logger: Logger, ) -> Result { + let mailbox = mailbox.filter(|_| !matches!(mailbox_kind, MailboxKind::PublicFolders)); let root = mailbox_kind.distinguished_root(); - let request = find_folder_body(FolderRef::Distinguished(root), Traversal::Deep); + let request = find_folder_body(FolderRef::distinguished(root, mailbox), Traversal::Deep); let resp = client.call(url, "FindFolder", &request)?; let parsed = parse_find_folder_response(&resp.body)?; if logger.enabled(LEVEL_PROGRESS) { @@ -96,8 +98,8 @@ pub fn plan_folders( (_, FolderClass::Skipped) => {} } } - let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, logger)?; - let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind)?; + let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, mailbox, logger)?; + let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind, mailbox)?; Ok(FolderPlan { mail, calendar, @@ -111,6 +113,7 @@ fn resolve_well_known_roles( client: &EwsClient, url: &str, mailbox_kind: MailboxKind, + mailbox: Option<&str>, logger: Logger, ) -> Result, EwsError> { if !matches!(mailbox_kind, MailboxKind::Primary) { @@ -118,7 +121,7 @@ fn resolve_well_known_roles( } let folders: Vec> = WELL_KNOWN_ROLES .iter() - .map(|(d, _)| FolderRef::Distinguished(*d)) + .map(|(d, _)| FolderRef::distinguished(*d, mailbox)) .collect(); let request = get_folder_body(&folders, FolderShape::IdOnly); let resp = match client.call(url, "GetFolder", &request) { @@ -154,9 +157,13 @@ fn resolve_root_folder_id( client: &EwsClient, url: &str, mailbox_kind: MailboxKind, + mailbox: Option<&str>, ) -> Result { let root = mailbox_kind.distinguished_root(); - let body = get_folder_body(&[FolderRef::Distinguished(root)], FolderShape::IdOnly); + let body = get_folder_body( + &[FolderRef::distinguished(root, mailbox)], + FolderShape::IdOnly, + ); let resp = client.call(url, "GetFolder", &body)?; let entries = parse_get_folder_response(&resp.body)?; Ok(entries diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs index e276c87..c70d28c 100644 --- a/src/sync/import_imap/coordinator.rs +++ b/src/sync/import_imap/coordinator.rs @@ -27,7 +27,9 @@ use crate::imap::retry::{ }; use crate::imap::transport::Connector; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::emailmeta::email_index_from_blob; +use crate::sync::emailmeta::{ + add_mime_keywords, email_index_and_mime_keywords, email_index_from_blob, +}; use crate::sync::keys::index_to_json; use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; @@ -52,6 +54,7 @@ pub(super) struct RunOpts { fetch_batch: usize, fetch_batch_bytes: u64, include_deleted: bool, + mime_keywords: bool, logger: Logger, } @@ -199,6 +202,7 @@ pub struct ImapImportConfig { pub subscribed_only: bool, pub automap: bool, pub include_deleted: bool, + pub mime_keywords: bool, pub fetch_batch: usize, /// Byte cap for one fetch chunk, by RFC822.SIZE; see `sync::batch`. pub fetch_batch_bytes: u64, @@ -418,6 +422,7 @@ fn run_into( fetch_batch: config.fetch_batch.max(1), fetch_batch_bytes: config.fetch_batch_bytes.max(1), include_deleted: config.include_deleted, + mime_keywords: config.mime_keywords, logger, }; @@ -741,6 +746,7 @@ fn reconcile_folder( fetch_batch, fetch_batch_bytes: _, include_deleted: _, + mime_keywords: _, logger, } = opts; @@ -1109,6 +1115,7 @@ fn insert_single_message( fetch_batch: _, fetch_batch_bytes: _, include_deleted, + mime_keywords, logger, } = opts; let folder = target.folder; @@ -1146,15 +1153,15 @@ fn insert_single_message( .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()), }; let blob_id = db::blobs::intern_blob(tx, body)?; - let message_match = index_to_json(&email_index_from_blob(body)); + let mut keywords = translation.keywords; + let index = if mime_keywords { + email_index_and_mime_keywords(body, &mut keywords) + } else { + email_index_from_blob(body) + }; + let message_match = index_to_json(&index); let mailbox_ids = Value::Array(vec![Value::from(mailbox_local)]); - let keywords = Value::Array( - translation - .keywords - .iter() - .map(|k| Value::String(k.clone())) - .collect(), - ); + let keywords = Value::Array(keywords.into_iter().map(Value::String).collect()); tx.execute( "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) VALUES (?1, ?2, ?3, ?4, ?5)", @@ -1188,6 +1195,7 @@ fn refresh_present_flags( fetch_batch, fetch_batch_bytes: _, include_deleted, + mime_keywords, logger: _, } = opts; let mut updated: u64 = 0; @@ -1210,23 +1218,19 @@ fn refresh_present_flags( if translation.has_deleted_flag && !include_deleted { continue; } - let Some((local_id, existing)) = - load_email_keywords(&tx, source_id, folder, uidvalidity, uid)? + let Some(stored) = load_email_keywords(&tx, source_id, folder, uidvalidity, uid)? else { continue; }; - if keyword_set_differs(&existing, &translation.keywords) { - let json = Value::Array( - translation - .keywords - .iter() - .cloned() - .map(Value::String) - .collect(), - ); + let mut keywords = translation.keywords; + if mime_keywords && let Some(blob) = db::blobs::blob_bytes(&tx, stored.blob_id)? { + add_mime_keywords(&blob, &mut keywords); + } + if keyword_set_differs(&stored.keywords, &keywords) { + let json = Value::Array(keywords.into_iter().map(Value::String).collect()); tx.execute( "UPDATE emails SET keywords = ?1 WHERE id = ?2", - params![json.to_string(), local_id], + params![json.to_string(), stored.local_id], )?; updated += 1; } @@ -1236,24 +1240,34 @@ fn refresh_present_flags( Ok(updated) } +struct StoredKeywords { + local_id: i64, + blob_id: i64, + keywords: Vec, +} + fn load_email_keywords( tx: &rusqlite::Transaction<'_>, source_id: i64, folder: &str, uidvalidity: u32, uid: u32, -) -> Result)>, Error> { +) -> Result, Error> { let Some(local_id) = db::imap_ids::local_for_email(tx, source_id, folder, uidvalidity, uid)? else { return Ok(None); }; - let kw_json: String = tx.query_row( - "SELECT keywords FROM emails WHERE id = ?1", + let (blob_id, kw_json): (i64, String) = tx.query_row( + "SELECT blob_id, keywords FROM emails WHERE id = ?1", params![local_id], - |r| r.get(0), + |r| Ok((r.get(0)?, r.get(1)?)), )?; - let kws: Vec = serde_json::from_str(&kw_json).unwrap_or_default(); - Ok(Some((local_id, kws))) + let keywords = serde_json::from_str(&kw_json).unwrap_or_default(); + Ok(Some(StoredKeywords { + local_id, + blob_id, + keywords, + })) } fn keyword_set_differs(a: &[String], b: &[String]) -> bool { diff --git a/src/sync/import_maildir/tree.rs b/src/sync/import_maildir/tree.rs index 2df8f63..ba8ee6b 100644 --- a/src/sync/import_maildir/tree.rs +++ b/src/sync/import_maildir/tree.rs @@ -5,15 +5,19 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::path::{Path, PathBuf}; +use std::{ + borrow::Cow, + path::{Path, PathBuf}, +}; use regex::Regex; -use crate::imap::automap; +use crate::imap::{automap, name::decode_mailbox_name}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedFolder { pub name: String, + pub decoded_name: String, pub leaf: String, pub parent_path: Option, pub path: PathBuf, @@ -57,6 +61,7 @@ pub fn discover(root: &Path, automap_enabled: bool) -> Result Result Option { name.rfind('.').map(|i| name[..i].to_owned()) } -/// A folder's name as shown: Maildir++ keeps names in modified UTF-7, as -/// Dovecot and Courier write them. A name that isn't valid modified UTF-7 is -/// shown as it is on disk. -fn display_leaf(raw: &str) -> String { - crate::imap::name::decode_mailbox_name(raw).unwrap_or_else(|_| raw.to_owned()) +fn decode_segment(segment: &str) -> Cow<'_, str> { + if !segment.contains('&') { + return Cow::Borrowed(segment); + } + decode_mailbox_name(segment).map_or(Cow::Borrowed(segment), Cow::Owned) +} + +fn decode_name(name: &str) -> String { + let mut decoded = String::with_capacity(name.len()); + for (i, segment) in name.split('.').enumerate() { + if i > 0 { + decoded.push('.'); + } + decoded.push_str(&decode_segment(segment)); + } + decoded } fn leaf_of(name: &str) -> &str { @@ -174,7 +191,7 @@ fn leaf_of(name: &str) -> &str { pub fn apply_filters(folders: Vec, filters: &FolderFilters) -> Vec { folders .into_iter() - .filter(|f| match_filters(&f.name, filters)) + .filter(|f| match_filters(&f.decoded_name, filters)) .collect() } @@ -187,11 +204,11 @@ pub fn restore_ephemeral_parents(folders: &mut Vec, root: &Path) if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) { continue; } - let leaf = display_leaf(leaf_of(&ancestor)); let parent_path = split_parent(&ancestor); to_add.push(ResolvedFolder { + decoded_name: decode_name(&ancestor), + leaf: decode_segment(leaf_of(&ancestor)).into_owned(), name: ancestor, - leaf, parent_path, path: root.to_path_buf(), role: None, @@ -386,9 +403,9 @@ mod tests { fn leaf_and_split_handle_root_and_nested() { assert_eq!(leaf_of("INBOX"), "INBOX"); assert_eq!(leaf_of("Archive.2025"), "2025"); - assert_eq!(display_leaf("&AMk-quipe &- co"), "Équipe & co"); - assert_eq!(display_leaf("Plain"), "Plain"); - assert_eq!(display_leaf("Équipe"), "Équipe"); + assert_eq!(decode_segment("&AMk-quipe &- co"), "Équipe & co"); + assert_eq!(decode_segment("Plain"), "Plain"); + assert_eq!(decode_segment("Équipe"), "Équipe"); assert_eq!(split_parent("INBOX"), None); assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned())); } @@ -398,6 +415,7 @@ mod tests { .iter() .map(|n| ResolvedFolder { name: (*n).to_owned(), + decoded_name: (*n).to_owned(), leaf: (*n).to_owned(), parent_path: None, path: PathBuf::new(), @@ -502,4 +520,61 @@ mod tests { ] ); } + + // Dovecot stores non-ASCII and '&' in folder names as modified UTF-7. + #[test] + fn discover_decodes_modified_utf7_names() { + let td = tempfile::tempdir().unwrap(); + make_maildir( + td.path(), + &[".&AOk-t&AOk-", ".R&-D", ".Plain", ".&ZeVnLIqe-.&U,BTFw-"], + ); + let folders = discover(td.path(), true).unwrap(); + let by_name = |name: &str| folders.iter().find(|f| f.name == name).unwrap(); + + let ete = by_name("&AOk-t&AOk-"); + assert_eq!(ete.decoded_name, "été"); + assert_eq!(ete.leaf, "été"); + assert_eq!(by_name("R&-D").leaf, "R&D"); + assert_eq!(by_name("Plain").leaf, "Plain"); + + let nested = by_name("&ZeVnLIqe-.&U,BTFw-"); + assert_eq!(nested.decoded_name, "日本語.台北"); + assert_eq!(nested.leaf, "台北"); + assert_eq!(nested.parent_path.as_deref(), Some("&ZeVnLIqe-")); + let parent = by_name("&ZeVnLIqe-"); + assert!(parent.ephemeral); + assert_eq!(parent.leaf, "日本語"); + } + + // Names that are not valid modified UTF-7 are kept byte for byte. + #[test] + fn decode_keeps_literal_ampersands_and_utf8() { + for name in ["R&D", "AT&T", "Tom & Jerry-2", "Été", "Sales&", "A&-B&C"] { + assert_eq!(decode_name(name), name); + } + assert_eq!(decode_name("R&D.&AOk-t&AOk-"), "R&D.été"); + } + + #[test] + fn filters_match_decoded_names() { + let mut folders = folders_for(&["INBOX", "Plain"]); + folders.push(ResolvedFolder { + name: "&AOk-t&AOk-".to_owned(), + decoded_name: "été".to_owned(), + leaf: "été".to_owned(), + parent_path: None, + path: PathBuf::new(), + role: None, + ephemeral: false, + }); + let filters = FolderFilters { + include: Vec::new(), + exclude: Vec::new(), + explicit: vec!["été".to_owned()], + }; + let kept = apply_filters(folders, &filters); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].name, "&AOk-t&AOk-"); + } } diff --git a/tests/integration_cyrus.rs b/tests/integration_cyrus.rs index feb6b24..8d743ba 100644 --- a/tests/integration_cyrus.rs +++ b/tests/integration_cyrus.rs @@ -37,6 +37,7 @@ fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig { subscribed_only: false, automap: true, include_deleted: false, + mime_keywords: false, fetch_batch: 64, fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, imap_connections: 2, diff --git a/tests/integration_dovecot.rs b/tests/integration_dovecot.rs index 17183fd..27061a3 100644 --- a/tests/integration_dovecot.rs +++ b/tests/integration_dovecot.rs @@ -42,6 +42,7 @@ fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportCon subscribed_only: false, automap: true, include_deleted: false, + mime_keywords: false, fetch_batch: 64, fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, imap_connections: 2, diff --git a/tests/integration_export_imap.rs b/tests/integration_export_imap.rs index a5470e2..bc79c14 100644 --- a/tests/integration_export_imap.rs +++ b/tests/integration_export_imap.rs @@ -183,6 +183,7 @@ fn dovecot_export_imap_round_trip_converges() { subscribed_only: false, automap: true, include_deleted: false, + mime_keywords: false, fetch_batch: 64, fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, imap_connections: 1, diff --git a/tests/mock_exchange_ews.rs b/tests/mock_exchange_ews.rs index 17463cf..539963c 100644 --- a/tests/mock_exchange_ews.rs +++ b/tests/mock_exchange_ews.rs @@ -1009,3 +1009,76 @@ fn get_folder_messages_preserve_position_when_one_errors() { assert!(!msgs[1].success); assert!(msgs[2].success); } + +fn plan_folders_against_mock( + kind: inbuxa_migrate::exchange_ews::types::MailboxKind, + mailbox: Option<&str>, + body_matcher: Matcher, +) { + use inbuxa_migrate::logging::Logger; + use inbuxa_migrate::sync::import_exchange_ews::folders::plan_folders; + + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let find_folder = envelope(&format!( + "\ + NoError\ + \ + " + )); + let get_folder = envelope(&format!( + "\ + \ + NoError\ + \ + " + )); + let find = server + .mock("POST", "/EWS/Exchange.asmx") + .match_header("soapaction", Matcher::Regex("/FindFolder\"$".into())) + .match_body(body_matcher.clone()) + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(find_folder) + .expect(1) + .create(); + let get = server + .mock("POST", "/EWS/Exchange.asmx") + .match_header("soapaction", Matcher::Regex("/GetFolder\"$".into())) + .match_body(body_matcher) + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(get_folder) + .expect_at_least(1) + .create(); + + let plan = plan_folders(&client(0), &url, kind, mailbox, Logger::new(0)) + .expect("folder planning should succeed"); + assert_eq!(plan.root_folder_id, "ROOT"); + find.assert(); + get.assert(); +} + +#[test] +fn explicit_mailbox_addresses_distinguished_folders_in_that_mailbox() { + // Regression for importing a FullAccess mailbox with Basic auth: folders must name --mailbox. + plan_folders_against_mock( + inbuxa_migrate::exchange_ews::types::MailboxKind::Primary, + Some("bob@corp.example"), + Matcher::Regex( + "bob@corp.example".into(), + ), + ); +} + +#[test] +fn public_folders_ignore_explicit_mailbox() { + // Public folder roots cannot carry a Mailbox element. + plan_folders_against_mock( + inbuxa_migrate::exchange_ews::types::MailboxKind::PublicFolders, + Some("bob@corp.example"), + Matcher::Regex( + "^(?s)[^@]*[^@]*$".into(), + ), + ); +} diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs index 0a1eaec..7d87713 100644 --- a/tests/mock_imap.rs +++ b/tests/mock_imap.rs @@ -170,6 +170,7 @@ fn run_import( subscribed_only: false, automap: true, include_deleted: false, + mime_keywords: false, fetch_batch: 256, fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, imap_connections: 1, @@ -922,6 +923,71 @@ fn coordinator_present_flag_change_updates_keywords() { ); } +const MSG_BODY_KEYWORDS: &[u8] = b"From: a@b\r\nTo: c@d\r\nSubject: hi\r\nMessage-ID: \r\nKeywords: bearbeitet von KL, VIP\r\nDate: Mon, 12 May 2025 10:00:00 +0000\r\n\r\nhello"; + +fn stored_keywords(archive: &PathBuf) -> Vec { + let dbc = Connection::open(archive).unwrap(); + let kw: String = dbc + .query_row("SELECT keywords FROM emails LIMIT 1", [], |r| r.get(0)) + .unwrap(); + let mut kws: Vec = serde_json::from_str(&kw).unwrap(); + kws.sort_unstable(); + kws +} + +// With --mime-keywords the Keywords header is imported and survives a later IMAP flag refresh. +#[test] +fn coordinator_mime_keywords_imported_and_kept_on_flag_refresh() { + let mut scripts: Vec