Merge upstream v1.0.15 #63

Merged
jcoffey-dev merged 2 commits from import/upstream-v1.0.15 into main 2026-11-09 06:54:21 +00:00
14 changed files with 515 additions and 57 deletions

No files matched your search

+19
View File
@@ -7,6 +7,25 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
## [Unreleased]
### Added
- IMAP: `--mime-keywords` adds the phrases of each message's `Keywords:`
header as keywords. Exchange shows its categories over IMAP this way.
Each phrase is lowercased and its spaces become `_`. Letters outside ASCII
are dropped, and a phrase can't name a system keyword such as `$seen`.
### Changed
- Upstream v1.0.15 is merged in.
### Fixed
- Maildir: folder names kept in modified UTF-7 are now decoded in full
(`.&ZeVnLIqe-.&U,BTFw-` is `日本語.台北`). Folder selection and role
guessing use the decoded name. Before, only the last part of a name was
decoded, for display.
- Exchange EWS: with Basic or delegated OAuth sign-in, `--mailbox` naming
another mailbox the user has full access to imported the user's own
mailbox instead. Folders are now addressed in the mailbox `--mailbox`
names.
## [27.1-rc.11]
The eleventh release candidate of the 27.1 series: upstream v1.0.14, and
+7
View File
@@ -346,6 +346,12 @@ struct ImapImportArgs {
#[arg(long, help = "Import \\Deleted messages with the $deleted keyword")]
include_deleted: bool,
#[arg(
long,
help = "Add the phrases of each message's Keywords: header (Exchange categories) as JMAP keywords"
)]
mime_keywords: bool,
#[arg(
long,
value_name = "N",
@@ -1391,6 +1397,7 @@ fn resolve_imap_import(args: ImapImportArgs) -> Result<Action, Error> {
subscribed_only: args.subscribed_only,
automap: !args.noautomap,
include_deleted: args.include_deleted,
mime_keywords: args.mime_keywords,
fetch_batch: args.fetch_batch,
fetch_batch_bytes: args.fetch_batch_mib.max(1).saturating_mul(1024 * 1024),
imap_connections,
+38
View File
@@ -11,9 +11,19 @@ use crate::exchange_ews::types::{DistinguishedFolderId, FolderId, ItemId, Server
#[derive(Debug, Clone, Copy)]
pub enum FolderRef<'a> {
Distinguished(DistinguishedFolderId),
DistinguishedIn(DistinguishedFolderId, &'a str),
Concrete(&'a FolderId),
}
impl<'a> FolderRef<'a> {
pub fn distinguished(id: DistinguishedFolderId, mailbox: Option<&'a str>) -> FolderRef<'a> {
match mailbox {
Some(mailbox) => FolderRef::DistinguishedIn(id, mailbox),
None => FolderRef::Distinguished(id),
}
}
}
#[derive(Debug, Clone, Copy)]
pub enum FolderShape {
IdOnly,
@@ -266,6 +276,13 @@ fn write_folder_ref(out: &mut String, parent: FolderRef<'_>) {
out.push_str(d.as_str());
out.push_str("\"/>");
}
FolderRef::DistinguishedIn(d, mailbox) => {
out.push_str("<t:DistinguishedFolderId Id=\"");
out.push_str(d.as_str());
out.push_str("\"><t:Mailbox><t:EmailAddress>");
write_escaped(out, mailbox);
out.push_str("</t:EmailAddress></t:Mailbox></t:DistinguishedFolderId>");
}
FolderRef::Concrete(f) => write_folder_id_element(out, f),
}
}
@@ -305,6 +322,27 @@ mod tests {
));
}
#[test]
fn distinguished_folder_in_named_mailbox_carries_escaped_address() {
let body = find_folder_body(
FolderRef::distinguished(
DistinguishedFolderId::MsgFolderRoot,
Some("b&[email protected]"),
),
Traversal::Deep,
);
assert!(body.contains(
"<t:DistinguishedFolderId Id=\"msgfolderroot\"><t:Mailbox>\
<t:EmailAddress>b&amp;[email protected]</t:EmailAddress></t:Mailbox>\
</t:DistinguishedFolderId>"
));
let body = find_folder_body(
FolderRef::distinguished(DistinguishedFolderId::MsgFolderRoot, None),
Traversal::Deep,
);
assert!(body.contains("<t:DistinguishedFolderId Id=\"msgfolderroot\"/>"));
}
#[test]
fn find_item_paginates_with_offset_and_page_size() {
let folder = FolderId::new("FID", "FCK");
+144 -3
View File
@@ -4,7 +4,9 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
use mail_parser::{Address, HeaderName, HeaderValue, MessageParser};
use std::slice;
use mail_parser::{Address, HeaderName, HeaderValue, Message, MessageParser};
use crate::sync::keys::{EmailIndex, email_index};
@@ -30,10 +32,78 @@ pub fn email_index_from_blob(bytes: &[u8]) -> EmailIndex {
email_meta_from_blob(bytes).0
}
pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option<String>) {
const MAX_KEYWORD_LEN: usize = 255;
const MAX_MIME_KEYWORDS: usize = 64;
pub fn email_index_and_mime_keywords(bytes: &[u8], keywords: &mut Vec<String>) -> EmailIndex {
let Some(m) = MessageParser::default().parse(bytes) else {
return (email_index(&[], &[], "", "", &[]), None);
return empty_meta().0;
};
add_keywords_header(&m, keywords);
email_meta(&m).0
}
pub fn add_mime_keywords(bytes: &[u8], keywords: &mut Vec<String>) {
if let Some(m) = MessageParser::default().parse_headers(bytes) {
add_keywords_header(&m, keywords);
}
}
fn add_keywords_header(m: &Message<'_>, keywords: &mut Vec<String>) {
let mut added = 0;
for value in m.header_values(HeaderName::Keywords) {
let phrases = match value {
HeaderValue::Text(phrase) => slice::from_ref(phrase),
HeaderValue::TextList(phrases) => phrases.as_slice(),
_ => &[],
};
for keyword in phrases.iter().filter_map(|phrase| jmap_keyword(phrase)) {
if !keywords.iter().any(|k| k.eq_ignore_ascii_case(&keyword)) {
keywords.push(keyword);
added += 1;
if added == MAX_MIME_KEYWORDS {
return;
}
}
}
}
}
fn jmap_keyword(phrase: &str) -> Option<String> {
let mut keyword = String::with_capacity(phrase.len().min(MAX_KEYWORD_LEN));
let mut separator = false;
for ch in phrase.chars() {
if ch.is_ascii_graphic()
&& !matches!(ch, '(' | ')' | '{' | ']' | '%' | '*' | '"' | '\\')
&& !(ch == '$' && keyword.is_empty())
{
let sep = separator && !keyword.is_empty();
if keyword.len() + usize::from(sep) >= MAX_KEYWORD_LEN {
break;
}
if sep {
keyword.push('_');
}
keyword.push(ch.to_ascii_lowercase());
separator = false;
} else {
separator = true;
}
}
(!keyword.is_empty()).then_some(keyword)
}
fn empty_meta() -> (EmailIndex, Option<String>) {
(email_index(&[], &[], "", "", &[]), None)
}
pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option<String>) {
MessageParser::default()
.parse(bytes)
.map_or_else(empty_meta, |m| email_meta(&m))
}
fn email_meta(m: &Message<'_>) -> (EmailIndex, Option<String>) {
let mut mids: Vec<String> = Vec::new();
for v in m.header_values(HeaderName::MessageId) {
mids.extend(message_ids(v));
@@ -59,6 +129,77 @@ mod tests {
use super::*;
use crate::sync::keys::{EmailKey, email_key};
// Every Keywords header and every comma-separated phrase is mapped to a JMAP keyword.
#[test]
fn mime_keywords_are_read_from_every_keywords_header() {
let mut keywords = vec!["$seen".to_owned(), "VIP".to_owned()];
let raw = b"Message-ID: <k@h>\r\nKeywords: bearbeitet von KL, Wiedervorlage\r\nKeywords: vip, =?UTF-8?Q?Pr=C3=BCfung?=\r\n\r\nbody";
let idx = email_index_and_mime_keywords(raw, &mut keywords);
assert_eq!(idx.mids, vec!["k@h".to_owned()]);
assert_eq!(
keywords,
vec![
"$seen",
"VIP",
"bearbeitet_von_kl",
"wiedervorlage",
"pr_fung"
]
);
let mut from_headers = vec!["$seen".to_owned(), "VIP".to_owned()];
add_mime_keywords(raw, &mut from_headers);
assert_eq!(from_headers, keywords);
}
// Phrases are reduced to the RFC 8621 keyword alphabet, without a leading "$", within 255 chars.
#[test]
fn jmap_keyword_follows_rfc8621_syntax() {
assert_eq!(
jmap_keyword(" Review / Legal ").as_deref(),
Some("review_/_legal")
);
assert_eq!(
jmap_keyword("a(b)c{d]e%f*g\"h\\i").as_deref(),
Some("a_b_c_d_e_f_g_h_i")
);
assert_eq!(jmap_keyword("$seen").as_deref(), Some("seen"));
assert_eq!(jmap_keyword(" $$Junk").as_deref(), Some("junk"));
assert_eq!(jmap_keyword("a$b").as_deref(), Some("a$b"));
assert_eq!(
jmap_keyword("Q4-2026.final_v2").as_deref(),
Some("q4-2026.final_v2")
);
assert_eq!(jmap_keyword(" ( ) "), None);
assert_eq!(
jmap_keyword("x".repeat(300).as_str()).map(|k| k.len()),
Some(255)
);
assert_eq!(
jmap_keyword(&format!("{} b", "a".repeat(254))).map(|k| k.len()),
Some(254)
);
}
// Empty phrases, messages without the header and the keyword count cap.
#[test]
fn mime_keywords_skip_empty_and_are_capped() {
let mut keywords = Vec::new();
add_mime_keywords(b"Keywords: , \r\n\r\nbody", &mut keywords);
add_mime_keywords(b"Subject: none\r\n\r\nbody", &mut keywords);
assert!(keywords.is_empty());
let many = (0..100)
.map(|n| format!("k{n}"))
.collect::<Vec<_>>()
.join(", ");
add_mime_keywords(
format!("Keywords: {many}\r\n\r\nbody").as_bytes(),
&mut keywords,
);
assert_eq!(keywords.len(), MAX_MIME_KEYWORDS);
}
#[test]
fn parses_core_fields_and_keys_by_message_id() {
let raw = b"From: Alice <[email protected]>\r\nTo: Bob <[email protected]>\r\nSubject: Hi there\r\nMessage-ID: <abc-1@host>\r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody\r\n";
+16 -4
View File
@@ -117,8 +117,14 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
let mut calendar_event_counts = TypeCounts::default();
let mut contact_counts = TypeCounts::default();
let plan =
plan_folders(&client, &session_url, config.mailbox_kind, logger).map_err(Error::from)?;
let plan = plan_folders(
&client,
&session_url,
config.mailbox_kind,
config.mailbox.as_deref(),
logger,
)
.map_err(Error::from)?;
folders::reconcile(
&mut conn,
@@ -188,8 +194,14 @@ fn run_dry(
config: &EwsImportConfig,
logger: crate::logging::Logger,
) -> Result<Summary, Error> {
let plan =
plan_folders(client, session_url, config.mailbox_kind, logger).map_err(Error::from)?;
let plan = plan_folders(
client,
session_url,
config.mailbox_kind,
config.mailbox.as_deref(),
logger,
)
.map_err(Error::from)?;
let mut summary = Summary::default();
let mut mailbox_counts = TypeCounts::default();
let mut calendar_counts = TypeCounts::default();
+12 -5
View File
@@ -41,10 +41,12 @@ pub fn plan_folders(
client: &EwsClient,
url: &str,
mailbox_kind: MailboxKind,
mailbox: Option<&str>,
logger: Logger,
) -> Result<FolderPlan, EwsError> {
let mailbox = mailbox.filter(|_| !matches!(mailbox_kind, MailboxKind::PublicFolders));
let root = mailbox_kind.distinguished_root();
let request = find_folder_body(FolderRef::Distinguished(root), Traversal::Deep);
let request = find_folder_body(FolderRef::distinguished(root, mailbox), Traversal::Deep);
let resp = client.call(url, "FindFolder", &request)?;
let parsed = parse_find_folder_response(&resp.body)?;
if logger.enabled(LEVEL_PROGRESS) {
@@ -96,8 +98,8 @@ pub fn plan_folders(
(_, FolderClass::Skipped) => {}
}
}
let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, logger)?;
let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind)?;
let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, mailbox, logger)?;
let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind, mailbox)?;
Ok(FolderPlan {
mail,
calendar,
@@ -111,6 +113,7 @@ fn resolve_well_known_roles(
client: &EwsClient,
url: &str,
mailbox_kind: MailboxKind,
mailbox: Option<&str>,
logger: Logger,
) -> Result<HashMap<String, &'static str>, EwsError> {
if !matches!(mailbox_kind, MailboxKind::Primary) {
@@ -118,7 +121,7 @@ fn resolve_well_known_roles(
}
let folders: Vec<FolderRef<'_>> = WELL_KNOWN_ROLES
.iter()
.map(|(d, _)| FolderRef::Distinguished(*d))
.map(|(d, _)| FolderRef::distinguished(*d, mailbox))
.collect();
let request = get_folder_body(&folders, FolderShape::IdOnly);
let resp = match client.call(url, "GetFolder", &request) {
@@ -154,9 +157,13 @@ fn resolve_root_folder_id(
client: &EwsClient,
url: &str,
mailbox_kind: MailboxKind,
mailbox: Option<&str>,
) -> Result<String, EwsError> {
let root = mailbox_kind.distinguished_root();
let body = get_folder_body(&[FolderRef::Distinguished(root)], FolderShape::IdOnly);
let body = get_folder_body(
&[FolderRef::distinguished(root, mailbox)],
FolderShape::IdOnly,
);
let resp = client.call(url, "GetFolder", &body)?;
let entries = parse_get_folder_response(&resp.body)?;
Ok(entries
+41 -27
View File
@@ -27,7 +27,9 @@ use crate::imap::retry::{
};
use crate::imap::transport::Connector;
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::sync::emailmeta::email_index_from_blob;
use crate::sync::emailmeta::{
add_mime_keywords, email_index_and_mime_keywords, email_index_from_blob,
};
use crate::sync::keys::index_to_json;
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
@@ -52,6 +54,7 @@ pub(super) struct RunOpts {
fetch_batch: usize,
fetch_batch_bytes: u64,
include_deleted: bool,
mime_keywords: bool,
logger: Logger,
}
@@ -199,6 +202,7 @@ pub struct ImapImportConfig {
pub subscribed_only: bool,
pub automap: bool,
pub include_deleted: bool,
pub mime_keywords: bool,
pub fetch_batch: usize,
/// Byte cap for one fetch chunk, by RFC822.SIZE; see `sync::batch`.
pub fetch_batch_bytes: u64,
@@ -418,6 +422,7 @@ fn run_into(
fetch_batch: config.fetch_batch.max(1),
fetch_batch_bytes: config.fetch_batch_bytes.max(1),
include_deleted: config.include_deleted,
mime_keywords: config.mime_keywords,
logger,
};
@@ -741,6 +746,7 @@ fn reconcile_folder(
fetch_batch,
fetch_batch_bytes: _,
include_deleted: _,
mime_keywords: _,
logger,
} = opts;
@@ -1109,6 +1115,7 @@ fn insert_single_message(
fetch_batch: _,
fetch_batch_bytes: _,
include_deleted,
mime_keywords,
logger,
} = opts;
let folder = target.folder;
@@ -1146,15 +1153,15 @@ fn insert_single_message(
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()),
};
let blob_id = db::blobs::intern_blob(tx, body)?;
let message_match = index_to_json(&email_index_from_blob(body));
let mut keywords = translation.keywords;
let index = if mime_keywords {
email_index_and_mime_keywords(body, &mut keywords)
} else {
email_index_from_blob(body)
};
let message_match = index_to_json(&index);
let mailbox_ids = Value::Array(vec![Value::from(mailbox_local)]);
let keywords = Value::Array(
translation
.keywords
.iter()
.map(|k| Value::String(k.clone()))
.collect(),
);
let keywords = Value::Array(keywords.into_iter().map(Value::String).collect());
tx.execute(
"INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match)
VALUES (?1, ?2, ?3, ?4, ?5)",
@@ -1188,6 +1195,7 @@ fn refresh_present_flags(
fetch_batch,
fetch_batch_bytes: _,
include_deleted,
mime_keywords,
logger: _,
} = opts;
let mut updated: u64 = 0;
@@ -1210,23 +1218,19 @@ fn refresh_present_flags(
if translation.has_deleted_flag && !include_deleted {
continue;
}
let Some((local_id, existing)) =
load_email_keywords(&tx, source_id, folder, uidvalidity, uid)?
let Some(stored) = load_email_keywords(&tx, source_id, folder, uidvalidity, uid)?
else {
continue;
};
if keyword_set_differs(&existing, &translation.keywords) {
let json = Value::Array(
translation
.keywords
.iter()
.cloned()
.map(Value::String)
.collect(),
);
let mut keywords = translation.keywords;
if mime_keywords && let Some(blob) = db::blobs::blob_bytes(&tx, stored.blob_id)? {
add_mime_keywords(&blob, &mut keywords);
}
if keyword_set_differs(&stored.keywords, &keywords) {
let json = Value::Array(keywords.into_iter().map(Value::String).collect());
tx.execute(
"UPDATE emails SET keywords = ?1 WHERE id = ?2",
params![json.to_string(), local_id],
params![json.to_string(), stored.local_id],
)?;
updated += 1;
}
@@ -1236,24 +1240,34 @@ fn refresh_present_flags(
Ok(updated)
}
struct StoredKeywords {
local_id: i64,
blob_id: i64,
keywords: Vec<String>,
}
fn load_email_keywords(
tx: &rusqlite::Transaction<'_>,
source_id: i64,
folder: &str,
uidvalidity: u32,
uid: u32,
) -> Result<Option<(i64, Vec<String>)>, Error> {
) -> Result<Option<StoredKeywords>, Error> {
let Some(local_id) = db::imap_ids::local_for_email(tx, source_id, folder, uidvalidity, uid)?
else {
return Ok(None);
};
let kw_json: String = tx.query_row(
"SELECT keywords FROM emails WHERE id = ?1",
let (blob_id, kw_json): (i64, String) = tx.query_row(
"SELECT blob_id, keywords FROM emails WHERE id = ?1",
params![local_id],
|r| r.get(0),
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
let kws: Vec<String> = serde_json::from_str(&kw_json).unwrap_or_default();
Ok(Some((local_id, kws)))
let keywords = serde_json::from_str(&kw_json).unwrap_or_default();
Ok(Some(StoredKeywords {
local_id,
blob_id,
keywords,
}))
}
fn keyword_set_differs(a: &[String], b: &[String]) -> bool {
+93 -18
View File
@@ -5,15 +5,19 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
use std::path::{Path, PathBuf};
use std::{
borrow::Cow,
path::{Path, PathBuf},
};
use regex::Regex;
use crate::imap::automap;
use crate::imap::{automap, name::decode_mailbox_name};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedFolder {
pub name: String,
pub decoded_name: String,
pub leaf: String,
pub parent_path: Option<String>,
pub path: PathBuf,
@@ -57,6 +61,7 @@ pub fn discover(root: &Path, automap_enabled: bool) -> Result<Vec<ResolvedFolder
}
let mut folders = vec![ResolvedFolder {
name: "INBOX".to_owned(),
decoded_name: "INBOX".to_owned(),
leaf: "INBOX".to_owned(),
parent_path: None,
path: root.to_path_buf(),
@@ -97,15 +102,16 @@ pub fn discover(root: &Path, automap_enabled: bool) -> Result<Vec<ResolvedFolder
if let Some(p) = parent_path.as_deref() {
ensure_ephemeral_parents(p, &mut known, &mut folders, root);
}
let leaf = leaf_of(&canonical);
let decoded_name = decode_name(&canonical);
let role = if automap_enabled {
automap::role_for_folder(&canonical, &[], "", true)
automap::role_for_folder(&decoded_name, &[], "", true)
} else {
None
};
folders.push(ResolvedFolder {
leaf: decode_segment(leaf_of(&canonical)).into_owned(),
name: canonical.clone(),
leaf: display_leaf(leaf),
decoded_name,
parent_path,
path,
role,
@@ -126,11 +132,11 @@ fn ensure_ephemeral_parents(
if known.iter().any(|n| n == &ancestor) {
continue;
}
let leaf = display_leaf(leaf_of(&ancestor));
let parent_path = split_parent(&ancestor);
folders.push(ResolvedFolder {
decoded_name: decode_name(&ancestor),
leaf: decode_segment(leaf_of(&ancestor)).into_owned(),
name: ancestor.clone(),
leaf,
parent_path,
path: root.to_path_buf(),
role: None,
@@ -157,11 +163,22 @@ fn split_parent(name: &str) -> Option<String> {
name.rfind('.').map(|i| name[..i].to_owned())
}
/// A folder's name as shown: Maildir++ keeps names in modified UTF-7, as
/// Dovecot and Courier write them. A name that isn't valid modified UTF-7 is
/// shown as it is on disk.
fn display_leaf(raw: &str) -> String {
crate::imap::name::decode_mailbox_name(raw).unwrap_or_else(|_| raw.to_owned())
fn decode_segment(segment: &str) -> Cow<'_, str> {
if !segment.contains('&') {
return Cow::Borrowed(segment);
}
decode_mailbox_name(segment).map_or(Cow::Borrowed(segment), Cow::Owned)
}
fn decode_name(name: &str) -> String {
let mut decoded = String::with_capacity(name.len());
for (i, segment) in name.split('.').enumerate() {
if i > 0 {
decoded.push('.');
}
decoded.push_str(&decode_segment(segment));
}
decoded
}
fn leaf_of(name: &str) -> &str {
@@ -174,7 +191,7 @@ fn leaf_of(name: &str) -> &str {
pub fn apply_filters(folders: Vec<ResolvedFolder>, filters: &FolderFilters) -> Vec<ResolvedFolder> {
folders
.into_iter()
.filter(|f| match_filters(&f.name, filters))
.filter(|f| match_filters(&f.decoded_name, filters))
.collect()
}
@@ -187,11 +204,11 @@ pub fn restore_ephemeral_parents(folders: &mut Vec<ResolvedFolder>, root: &Path)
if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) {
continue;
}
let leaf = display_leaf(leaf_of(&ancestor));
let parent_path = split_parent(&ancestor);
to_add.push(ResolvedFolder {
decoded_name: decode_name(&ancestor),
leaf: decode_segment(leaf_of(&ancestor)).into_owned(),
name: ancestor,
leaf,
parent_path,
path: root.to_path_buf(),
role: None,
@@ -386,9 +403,9 @@ mod tests {
fn leaf_and_split_handle_root_and_nested() {
assert_eq!(leaf_of("INBOX"), "INBOX");
assert_eq!(leaf_of("Archive.2025"), "2025");
assert_eq!(display_leaf("&AMk-quipe &- co"), "Équipe & co");
assert_eq!(display_leaf("Plain"), "Plain");
assert_eq!(display_leaf("Équipe"), "Équipe");
assert_eq!(decode_segment("&AMk-quipe &- co"), "Équipe & co");
assert_eq!(decode_segment("Plain"), "Plain");
assert_eq!(decode_segment("Équipe"), "Équipe");
assert_eq!(split_parent("INBOX"), None);
assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned()));
}
@@ -398,6 +415,7 @@ mod tests {
.iter()
.map(|n| ResolvedFolder {
name: (*n).to_owned(),
decoded_name: (*n).to_owned(),
leaf: (*n).to_owned(),
parent_path: None,
path: PathBuf::new(),
@@ -502,4 +520,61 @@ mod tests {
]
);
}
// Dovecot stores non-ASCII and '&' in folder names as modified UTF-7.
#[test]
fn discover_decodes_modified_utf7_names() {
let td = tempfile::tempdir().unwrap();
make_maildir(
td.path(),
&[".&AOk-t&AOk-", ".R&-D", ".Plain", ".&ZeVnLIqe-.&U,BTFw-"],
);
let folders = discover(td.path(), true).unwrap();
let by_name = |name: &str| folders.iter().find(|f| f.name == name).unwrap();
let ete = by_name("&AOk-t&AOk-");
assert_eq!(ete.decoded_name, "été");
assert_eq!(ete.leaf, "été");
assert_eq!(by_name("R&-D").leaf, "R&D");
assert_eq!(by_name("Plain").leaf, "Plain");
let nested = by_name("&ZeVnLIqe-.&U,BTFw-");
assert_eq!(nested.decoded_name, "日本語.台北");
assert_eq!(nested.leaf, "台北");
assert_eq!(nested.parent_path.as_deref(), Some("&ZeVnLIqe-"));
let parent = by_name("&ZeVnLIqe-");
assert!(parent.ephemeral);
assert_eq!(parent.leaf, "日本語");
}
// Names that are not valid modified UTF-7 are kept byte for byte.
#[test]
fn decode_keeps_literal_ampersands_and_utf8() {
for name in ["R&D", "AT&T", "Tom & Jerry-2", "Été", "Sales&", "A&-B&C"] {
assert_eq!(decode_name(name), name);
}
assert_eq!(decode_name("R&D.&AOk-t&AOk-"), "R&D.été");
}
#[test]
fn filters_match_decoded_names() {
let mut folders = folders_for(&["INBOX", "Plain"]);
folders.push(ResolvedFolder {
name: "&AOk-t&AOk-".to_owned(),
decoded_name: "été".to_owned(),
leaf: "été".to_owned(),
parent_path: None,
path: PathBuf::new(),
role: None,
ephemeral: false,
});
let filters = FolderFilters {
include: Vec::new(),
exclude: Vec::new(),
explicit: vec!["été".to_owned()],
};
let kept = apply_filters(folders, &filters);
assert_eq!(kept.len(), 1);
assert_eq!(kept[0].name, "&AOk-t&AOk-");
}
}
+1
View File
@@ -37,6 +37,7 @@ fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig {
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 64,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 2,
+1
View File
@@ -42,6 +42,7 @@ fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportCon
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 64,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 2,
+1
View File
@@ -183,6 +183,7 @@ fn dovecot_export_imap_round_trip_converges() {
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 64,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 1,
+73
View File
@@ -1009,3 +1009,76 @@ fn get_folder_messages_preserve_position_when_one_errors() {
assert!(!msgs[1].success);
assert!(msgs[2].success);
}
fn plan_folders_against_mock(
kind: inbuxa_migrate::exchange_ews::types::MailboxKind,
mailbox: Option<&str>,
body_matcher: Matcher,
) {
use inbuxa_migrate::logging::Logger;
use inbuxa_migrate::sync::import_exchange_ews::folders::plan_folders;
let mut server = mockito::Server::new();
let url = format!("{}/EWS/Exchange.asmx", server.url());
let find_folder = envelope(&format!(
"<m:FindFolderResponse{NS}><m:ResponseMessages><m:FindFolderResponseMessage ResponseClass=\"Success\">\
<m:ResponseCode>NoError</m:ResponseCode>\
<m:RootFolder TotalItemsInView=\"0\" IncludesLastItemInRange=\"true\"><t:Folders/></m:RootFolder>\
</m:FindFolderResponseMessage></m:ResponseMessages></m:FindFolderResponse>"
));
let get_folder = envelope(&format!(
"<m:GetFolderResponse{NS}><m:ResponseMessages>\
<m:GetFolderResponseMessage ResponseClass=\"Success\">\
<m:ResponseCode>NoError</m:ResponseCode>\
<m:Folders><t:Folder><t:FolderId Id=\"ROOT\" ChangeKey=\"K\"/></t:Folder></m:Folders>\
</m:GetFolderResponseMessage></m:ResponseMessages></m:GetFolderResponse>"
));
let find = server
.mock("POST", "/EWS/Exchange.asmx")
.match_header("soapaction", Matcher::Regex("/FindFolder\"$".into()))
.match_body(body_matcher.clone())
.with_status(200)
.with_header("content-type", TXT_XML)
.with_body(find_folder)
.expect(1)
.create();
let get = server
.mock("POST", "/EWS/Exchange.asmx")
.match_header("soapaction", Matcher::Regex("/GetFolder\"$".into()))
.match_body(body_matcher)
.with_status(200)
.with_header("content-type", TXT_XML)
.with_body(get_folder)
.expect_at_least(1)
.create();
let plan = plan_folders(&client(0), &url, kind, mailbox, Logger::new(0))
.expect("folder planning should succeed");
assert_eq!(plan.root_folder_id, "ROOT");
find.assert();
get.assert();
}
#[test]
fn explicit_mailbox_addresses_distinguished_folders_in_that_mailbox() {
// Regression for importing a FullAccess mailbox with Basic auth: folders must name --mailbox.
plan_folders_against_mock(
inbuxa_migrate::exchange_ews::types::MailboxKind::Primary,
Some("[email protected]"),
Matcher::Regex(
"<t:DistinguishedFolderId Id=\"[a-z]+\"><t:Mailbox><t:EmailAddress>[email protected]</t:EmailAddress></t:Mailbox></t:DistinguishedFolderId>".into(),
),
);
}
#[test]
fn public_folders_ignore_explicit_mailbox() {
// Public folder roots cannot carry a Mailbox element.
plan_folders_against_mock(
inbuxa_migrate::exchange_ews::types::MailboxKind::PublicFolders,
Some("[email protected]"),
Matcher::Regex(
"^(?s)[^@]*<t:DistinguishedFolderId Id=\"publicfoldersroot\"/>[^@]*$".into(),
),
);
}
+67
View File
@@ -170,6 +170,7 @@ fn run_import(
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 256,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 1,
@@ -922,6 +923,71 @@ fn coordinator_present_flag_change_updates_keywords() {
);
}
const MSG_BODY_KEYWORDS: &[u8] = b"From: a@b\r\nTo: c@d\r\nSubject: hi\r\nMessage-ID: <m2@h>\r\nKeywords: bearbeitet von KL, VIP\r\nDate: Mon, 12 May 2025 10:00:00 +0000\r\n\r\nhello";
fn stored_keywords(archive: &PathBuf) -> Vec<String> {
let dbc = Connection::open(archive).unwrap();
let kw: String = dbc
.query_row("SELECT keywords FROM emails LIMIT 1", [], |r| r.get(0))
.unwrap();
let mut kws: Vec<String> = serde_json::from_str(&kw).unwrap();
kws.sort_unstable();
kws
}
// With --mime-keywords the Keywords header is imported and survives a later IMAP flag refresh.
#[test]
fn coordinator_mime_keywords_imported_and_kept_on_flag_refresh() {
let mut scripts: Vec<Script> = Vec::new();
scripts.extend(single_inbox_scripts(100, 2, MSG_BODY_KEYWORDS));
scripts.push(control_script_present_flags(
100,
2,
&[1],
&[(1, "\\Seen \\Flagged")],
));
scripts.push(worker_idle_script("IMAP4rev2 LITERAL+ AUTH=PLAIN"));
let server = MockImap::start_scripts(scripts);
let archive = tempfile("mime_keywords");
run_import(&server, "alice", archive.clone(), |c| {
c.mime_keywords = true
})
.expect("first import");
let first = stored_keywords(&archive);
assert!(
first.contains(&"bearbeitet_von_kl".to_owned()) && first.contains(&"vip".to_owned()),
"{first:?}"
);
let summary = run_import(&server, "alice", archive.clone(), |c| {
c.mime_keywords = true
})
.expect("second import");
let email = summary
.per_type
.iter()
.find(|(k, _)| *k == "email")
.unwrap();
assert_eq!(email.1.created, 0);
assert_eq!(email.1.updated, 1);
let mut expected = first;
expected.push("$flagged".to_owned());
expected.sort_unstable();
assert_eq!(stored_keywords(&archive), expected);
}
// Without --mime-keywords the Keywords header is ignored.
#[test]
fn coordinator_ignores_mime_keywords_by_default() {
let server = MockImap::start_scripts(single_inbox_scripts(100, 2, MSG_BODY_KEYWORDS));
let archive = tempfile("mime_keywords_off");
run_import(&server, "alice", archive.clone(), |_| {}).expect("import");
let kws = stored_keywords(&archive);
assert!(
!kws.iter().any(|k| k == "vip" || k == "bearbeitet_von_kl"),
"{kws:?}"
);
}
#[test]
fn coordinator_present_newly_deleted_is_left_intact() {
let mut scripts: Vec<Script> = Vec::new();
@@ -2625,6 +2691,7 @@ fn dry_run_never_selects_noselect_ancestors() {
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 256,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 1,
+2
View File
@@ -74,6 +74,7 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig {
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 256,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 4,
@@ -97,6 +98,7 @@ fn dovecot_config(account: &Account, imap: &integration::Endpoint) -> ImapImport
subscribed_only: false,
automap: true,
include_deleted: false,
mime_keywords: false,
fetch_batch: 64,
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
imap_connections: 2,