Merge upstream v1.0.15 #63
No files matched your search
@@ -7,6 +7,25 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- IMAP: `--mime-keywords` adds the phrases of each message's `Keywords:`
|
||||
header as keywords. Exchange shows its categories over IMAP this way.
|
||||
Each phrase is lowercased and its spaces become `_`. Letters outside ASCII
|
||||
are dropped, and a phrase can't name a system keyword such as `$seen`.
|
||||
|
||||
### Changed
|
||||
- Upstream v1.0.15 is merged in.
|
||||
|
||||
### Fixed
|
||||
- Maildir: folder names kept in modified UTF-7 are now decoded in full
|
||||
(`.&ZeVnLIqe-.&U,BTFw-` is `日本語.台北`). Folder selection and role
|
||||
guessing use the decoded name. Before, only the last part of a name was
|
||||
decoded, for display.
|
||||
- Exchange EWS: with Basic or delegated OAuth sign-in, `--mailbox` naming
|
||||
another mailbox the user has full access to imported the user's own
|
||||
mailbox instead. Folders are now addressed in the mailbox `--mailbox`
|
||||
names.
|
||||
|
||||
## [27.1-rc.11]
|
||||
|
||||
The eleventh release candidate of the 27.1 series: upstream v1.0.14, and
|
||||
|
||||
@@ -346,6 +346,12 @@ struct ImapImportArgs {
|
||||
#[arg(long, help = "Import \\Deleted messages with the $deleted keyword")]
|
||||
include_deleted: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
help = "Add the phrases of each message's Keywords: header (Exchange categories) as JMAP keywords"
|
||||
)]
|
||||
mime_keywords: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "N",
|
||||
@@ -1391,6 +1397,7 @@ fn resolve_imap_import(args: ImapImportArgs) -> Result<Action, Error> {
|
||||
subscribed_only: args.subscribed_only,
|
||||
automap: !args.noautomap,
|
||||
include_deleted: args.include_deleted,
|
||||
mime_keywords: args.mime_keywords,
|
||||
fetch_batch: args.fetch_batch,
|
||||
fetch_batch_bytes: args.fetch_batch_mib.max(1).saturating_mul(1024 * 1024),
|
||||
imap_connections,
|
||||
|
||||
@@ -11,9 +11,19 @@ use crate::exchange_ews::types::{DistinguishedFolderId, FolderId, ItemId, Server
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum FolderRef<'a> {
|
||||
Distinguished(DistinguishedFolderId),
|
||||
DistinguishedIn(DistinguishedFolderId, &'a str),
|
||||
Concrete(&'a FolderId),
|
||||
}
|
||||
|
||||
impl<'a> FolderRef<'a> {
|
||||
pub fn distinguished(id: DistinguishedFolderId, mailbox: Option<&'a str>) -> FolderRef<'a> {
|
||||
match mailbox {
|
||||
Some(mailbox) => FolderRef::DistinguishedIn(id, mailbox),
|
||||
None => FolderRef::Distinguished(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum FolderShape {
|
||||
IdOnly,
|
||||
@@ -266,6 +276,13 @@ fn write_folder_ref(out: &mut String, parent: FolderRef<'_>) {
|
||||
out.push_str(d.as_str());
|
||||
out.push_str("\"/>");
|
||||
}
|
||||
FolderRef::DistinguishedIn(d, mailbox) => {
|
||||
out.push_str("<t:DistinguishedFolderId Id=\"");
|
||||
out.push_str(d.as_str());
|
||||
out.push_str("\"><t:Mailbox><t:EmailAddress>");
|
||||
write_escaped(out, mailbox);
|
||||
out.push_str("</t:EmailAddress></t:Mailbox></t:DistinguishedFolderId>");
|
||||
}
|
||||
FolderRef::Concrete(f) => write_folder_id_element(out, f),
|
||||
}
|
||||
}
|
||||
@@ -305,6 +322,27 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinguished_folder_in_named_mailbox_carries_escaped_address() {
|
||||
let body = find_folder_body(
|
||||
FolderRef::distinguished(
|
||||
DistinguishedFolderId::MsgFolderRoot,
|
||||
Some("b&[email protected]"),
|
||||
),
|
||||
Traversal::Deep,
|
||||
);
|
||||
assert!(body.contains(
|
||||
"<t:DistinguishedFolderId Id=\"msgfolderroot\"><t:Mailbox>\
|
||||
<t:EmailAddress>b&[email protected]</t:EmailAddress></t:Mailbox>\
|
||||
</t:DistinguishedFolderId>"
|
||||
));
|
||||
let body = find_folder_body(
|
||||
FolderRef::distinguished(DistinguishedFolderId::MsgFolderRoot, None),
|
||||
Traversal::Deep,
|
||||
);
|
||||
assert!(body.contains("<t:DistinguishedFolderId Id=\"msgfolderroot\"/>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_item_paginates_with_offset_and_page_size() {
|
||||
let folder = FolderId::new("FID", "FCK");
|
||||
|
||||
+144
-3
@@ -4,7 +4,9 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use mail_parser::{Address, HeaderName, HeaderValue, MessageParser};
|
||||
use std::slice;
|
||||
|
||||
use mail_parser::{Address, HeaderName, HeaderValue, Message, MessageParser};
|
||||
|
||||
use crate::sync::keys::{EmailIndex, email_index};
|
||||
|
||||
@@ -30,10 +32,78 @@ pub fn email_index_from_blob(bytes: &[u8]) -> EmailIndex {
|
||||
email_meta_from_blob(bytes).0
|
||||
}
|
||||
|
||||
pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option<String>) {
|
||||
const MAX_KEYWORD_LEN: usize = 255;
|
||||
const MAX_MIME_KEYWORDS: usize = 64;
|
||||
|
||||
pub fn email_index_and_mime_keywords(bytes: &[u8], keywords: &mut Vec<String>) -> EmailIndex {
|
||||
let Some(m) = MessageParser::default().parse(bytes) else {
|
||||
return (email_index(&[], &[], "", "", &[]), None);
|
||||
return empty_meta().0;
|
||||
};
|
||||
add_keywords_header(&m, keywords);
|
||||
email_meta(&m).0
|
||||
}
|
||||
|
||||
pub fn add_mime_keywords(bytes: &[u8], keywords: &mut Vec<String>) {
|
||||
if let Some(m) = MessageParser::default().parse_headers(bytes) {
|
||||
add_keywords_header(&m, keywords);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_keywords_header(m: &Message<'_>, keywords: &mut Vec<String>) {
|
||||
let mut added = 0;
|
||||
for value in m.header_values(HeaderName::Keywords) {
|
||||
let phrases = match value {
|
||||
HeaderValue::Text(phrase) => slice::from_ref(phrase),
|
||||
HeaderValue::TextList(phrases) => phrases.as_slice(),
|
||||
_ => &[],
|
||||
};
|
||||
for keyword in phrases.iter().filter_map(|phrase| jmap_keyword(phrase)) {
|
||||
if !keywords.iter().any(|k| k.eq_ignore_ascii_case(&keyword)) {
|
||||
keywords.push(keyword);
|
||||
added += 1;
|
||||
if added == MAX_MIME_KEYWORDS {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn jmap_keyword(phrase: &str) -> Option<String> {
|
||||
let mut keyword = String::with_capacity(phrase.len().min(MAX_KEYWORD_LEN));
|
||||
let mut separator = false;
|
||||
for ch in phrase.chars() {
|
||||
if ch.is_ascii_graphic()
|
||||
&& !matches!(ch, '(' | ')' | '{' | ']' | '%' | '*' | '"' | '\\')
|
||||
&& !(ch == '$' && keyword.is_empty())
|
||||
{
|
||||
let sep = separator && !keyword.is_empty();
|
||||
if keyword.len() + usize::from(sep) >= MAX_KEYWORD_LEN {
|
||||
break;
|
||||
}
|
||||
if sep {
|
||||
keyword.push('_');
|
||||
}
|
||||
keyword.push(ch.to_ascii_lowercase());
|
||||
separator = false;
|
||||
} else {
|
||||
separator = true;
|
||||
}
|
||||
}
|
||||
(!keyword.is_empty()).then_some(keyword)
|
||||
}
|
||||
|
||||
fn empty_meta() -> (EmailIndex, Option<String>) {
|
||||
(email_index(&[], &[], "", "", &[]), None)
|
||||
}
|
||||
|
||||
pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option<String>) {
|
||||
MessageParser::default()
|
||||
.parse(bytes)
|
||||
.map_or_else(empty_meta, |m| email_meta(&m))
|
||||
}
|
||||
|
||||
fn email_meta(m: &Message<'_>) -> (EmailIndex, Option<String>) {
|
||||
let mut mids: Vec<String> = Vec::new();
|
||||
for v in m.header_values(HeaderName::MessageId) {
|
||||
mids.extend(message_ids(v));
|
||||
@@ -59,6 +129,77 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::sync::keys::{EmailKey, email_key};
|
||||
|
||||
// Every Keywords header and every comma-separated phrase is mapped to a JMAP keyword.
|
||||
#[test]
|
||||
fn mime_keywords_are_read_from_every_keywords_header() {
|
||||
let mut keywords = vec!["$seen".to_owned(), "VIP".to_owned()];
|
||||
let raw = b"Message-ID: <k@h>\r\nKeywords: bearbeitet von KL, Wiedervorlage\r\nKeywords: vip, =?UTF-8?Q?Pr=C3=BCfung?=\r\n\r\nbody";
|
||||
let idx = email_index_and_mime_keywords(raw, &mut keywords);
|
||||
assert_eq!(idx.mids, vec!["k@h".to_owned()]);
|
||||
assert_eq!(
|
||||
keywords,
|
||||
vec![
|
||||
"$seen",
|
||||
"VIP",
|
||||
"bearbeitet_von_kl",
|
||||
"wiedervorlage",
|
||||
"pr_fung"
|
||||
]
|
||||
);
|
||||
|
||||
let mut from_headers = vec!["$seen".to_owned(), "VIP".to_owned()];
|
||||
add_mime_keywords(raw, &mut from_headers);
|
||||
assert_eq!(from_headers, keywords);
|
||||
}
|
||||
|
||||
// Phrases are reduced to the RFC 8621 keyword alphabet, without a leading "$", within 255 chars.
|
||||
#[test]
|
||||
fn jmap_keyword_follows_rfc8621_syntax() {
|
||||
assert_eq!(
|
||||
jmap_keyword(" Review / Legal ").as_deref(),
|
||||
Some("review_/_legal")
|
||||
);
|
||||
assert_eq!(
|
||||
jmap_keyword("a(b)c{d]e%f*g\"h\\i").as_deref(),
|
||||
Some("a_b_c_d_e_f_g_h_i")
|
||||
);
|
||||
assert_eq!(jmap_keyword("$seen").as_deref(), Some("seen"));
|
||||
assert_eq!(jmap_keyword(" $$Junk").as_deref(), Some("junk"));
|
||||
assert_eq!(jmap_keyword("a$b").as_deref(), Some("a$b"));
|
||||
assert_eq!(
|
||||
jmap_keyword("Q4-2026.final_v2").as_deref(),
|
||||
Some("q4-2026.final_v2")
|
||||
);
|
||||
assert_eq!(jmap_keyword(" ( ) "), None);
|
||||
assert_eq!(
|
||||
jmap_keyword("x".repeat(300).as_str()).map(|k| k.len()),
|
||||
Some(255)
|
||||
);
|
||||
assert_eq!(
|
||||
jmap_keyword(&format!("{} b", "a".repeat(254))).map(|k| k.len()),
|
||||
Some(254)
|
||||
);
|
||||
}
|
||||
|
||||
// Empty phrases, messages without the header and the keyword count cap.
|
||||
#[test]
|
||||
fn mime_keywords_skip_empty_and_are_capped() {
|
||||
let mut keywords = Vec::new();
|
||||
add_mime_keywords(b"Keywords: , \r\n\r\nbody", &mut keywords);
|
||||
add_mime_keywords(b"Subject: none\r\n\r\nbody", &mut keywords);
|
||||
assert!(keywords.is_empty());
|
||||
|
||||
let many = (0..100)
|
||||
.map(|n| format!("k{n}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
add_mime_keywords(
|
||||
format!("Keywords: {many}\r\n\r\nbody").as_bytes(),
|
||||
&mut keywords,
|
||||
);
|
||||
assert_eq!(keywords.len(), MAX_MIME_KEYWORDS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_core_fields_and_keys_by_message_id() {
|
||||
let raw = b"From: Alice <[email protected]>\r\nTo: Bob <[email protected]>\r\nSubject: Hi there\r\nMessage-ID: <abc-1@host>\r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody\r\n";
|
||||
|
||||
@@ -117,8 +117,14 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
let mut calendar_event_counts = TypeCounts::default();
|
||||
let mut contact_counts = TypeCounts::default();
|
||||
|
||||
let plan =
|
||||
plan_folders(&client, &session_url, config.mailbox_kind, logger).map_err(Error::from)?;
|
||||
let plan = plan_folders(
|
||||
&client,
|
||||
&session_url,
|
||||
config.mailbox_kind,
|
||||
config.mailbox.as_deref(),
|
||||
logger,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
|
||||
folders::reconcile(
|
||||
&mut conn,
|
||||
@@ -188,8 +194,14 @@ fn run_dry(
|
||||
config: &EwsImportConfig,
|
||||
logger: crate::logging::Logger,
|
||||
) -> Result<Summary, Error> {
|
||||
let plan =
|
||||
plan_folders(client, session_url, config.mailbox_kind, logger).map_err(Error::from)?;
|
||||
let plan = plan_folders(
|
||||
client,
|
||||
session_url,
|
||||
config.mailbox_kind,
|
||||
config.mailbox.as_deref(),
|
||||
logger,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
let mut summary = Summary::default();
|
||||
let mut mailbox_counts = TypeCounts::default();
|
||||
let mut calendar_counts = TypeCounts::default();
|
||||
|
||||
@@ -41,10 +41,12 @@ pub fn plan_folders(
|
||||
client: &EwsClient,
|
||||
url: &str,
|
||||
mailbox_kind: MailboxKind,
|
||||
mailbox: Option<&str>,
|
||||
logger: Logger,
|
||||
) -> Result<FolderPlan, EwsError> {
|
||||
let mailbox = mailbox.filter(|_| !matches!(mailbox_kind, MailboxKind::PublicFolders));
|
||||
let root = mailbox_kind.distinguished_root();
|
||||
let request = find_folder_body(FolderRef::Distinguished(root), Traversal::Deep);
|
||||
let request = find_folder_body(FolderRef::distinguished(root, mailbox), Traversal::Deep);
|
||||
let resp = client.call(url, "FindFolder", &request)?;
|
||||
let parsed = parse_find_folder_response(&resp.body)?;
|
||||
if logger.enabled(LEVEL_PROGRESS) {
|
||||
@@ -96,8 +98,8 @@ pub fn plan_folders(
|
||||
(_, FolderClass::Skipped) => {}
|
||||
}
|
||||
}
|
||||
let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, logger)?;
|
||||
let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind)?;
|
||||
let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, mailbox, logger)?;
|
||||
let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind, mailbox)?;
|
||||
Ok(FolderPlan {
|
||||
mail,
|
||||
calendar,
|
||||
@@ -111,6 +113,7 @@ fn resolve_well_known_roles(
|
||||
client: &EwsClient,
|
||||
url: &str,
|
||||
mailbox_kind: MailboxKind,
|
||||
mailbox: Option<&str>,
|
||||
logger: Logger,
|
||||
) -> Result<HashMap<String, &'static str>, EwsError> {
|
||||
if !matches!(mailbox_kind, MailboxKind::Primary) {
|
||||
@@ -118,7 +121,7 @@ fn resolve_well_known_roles(
|
||||
}
|
||||
let folders: Vec<FolderRef<'_>> = WELL_KNOWN_ROLES
|
||||
.iter()
|
||||
.map(|(d, _)| FolderRef::Distinguished(*d))
|
||||
.map(|(d, _)| FolderRef::distinguished(*d, mailbox))
|
||||
.collect();
|
||||
let request = get_folder_body(&folders, FolderShape::IdOnly);
|
||||
let resp = match client.call(url, "GetFolder", &request) {
|
||||
@@ -154,9 +157,13 @@ fn resolve_root_folder_id(
|
||||
client: &EwsClient,
|
||||
url: &str,
|
||||
mailbox_kind: MailboxKind,
|
||||
mailbox: Option<&str>,
|
||||
) -> Result<String, EwsError> {
|
||||
let root = mailbox_kind.distinguished_root();
|
||||
let body = get_folder_body(&[FolderRef::Distinguished(root)], FolderShape::IdOnly);
|
||||
let body = get_folder_body(
|
||||
&[FolderRef::distinguished(root, mailbox)],
|
||||
FolderShape::IdOnly,
|
||||
);
|
||||
let resp = client.call(url, "GetFolder", &body)?;
|
||||
let entries = parse_get_folder_response(&resp.body)?;
|
||||
Ok(entries
|
||||
|
||||
@@ -27,7 +27,9 @@ use crate::imap::retry::{
|
||||
};
|
||||
use crate::imap::transport::Connector;
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::emailmeta::email_index_from_blob;
|
||||
use crate::sync::emailmeta::{
|
||||
add_mime_keywords, email_index_and_mime_keywords, email_index_from_blob,
|
||||
};
|
||||
use crate::sync::keys::index_to_json;
|
||||
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
|
||||
@@ -52,6 +54,7 @@ pub(super) struct RunOpts {
|
||||
fetch_batch: usize,
|
||||
fetch_batch_bytes: u64,
|
||||
include_deleted: bool,
|
||||
mime_keywords: bool,
|
||||
logger: Logger,
|
||||
}
|
||||
|
||||
@@ -199,6 +202,7 @@ pub struct ImapImportConfig {
|
||||
pub subscribed_only: bool,
|
||||
pub automap: bool,
|
||||
pub include_deleted: bool,
|
||||
pub mime_keywords: bool,
|
||||
pub fetch_batch: usize,
|
||||
/// Byte cap for one fetch chunk, by RFC822.SIZE; see `sync::batch`.
|
||||
pub fetch_batch_bytes: u64,
|
||||
@@ -418,6 +422,7 @@ fn run_into(
|
||||
fetch_batch: config.fetch_batch.max(1),
|
||||
fetch_batch_bytes: config.fetch_batch_bytes.max(1),
|
||||
include_deleted: config.include_deleted,
|
||||
mime_keywords: config.mime_keywords,
|
||||
logger,
|
||||
};
|
||||
|
||||
@@ -741,6 +746,7 @@ fn reconcile_folder(
|
||||
fetch_batch,
|
||||
fetch_batch_bytes: _,
|
||||
include_deleted: _,
|
||||
mime_keywords: _,
|
||||
logger,
|
||||
} = opts;
|
||||
|
||||
@@ -1109,6 +1115,7 @@ fn insert_single_message(
|
||||
fetch_batch: _,
|
||||
fetch_batch_bytes: _,
|
||||
include_deleted,
|
||||
mime_keywords,
|
||||
logger,
|
||||
} = opts;
|
||||
let folder = target.folder;
|
||||
@@ -1146,15 +1153,15 @@ fn insert_single_message(
|
||||
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()),
|
||||
};
|
||||
let blob_id = db::blobs::intern_blob(tx, body)?;
|
||||
let message_match = index_to_json(&email_index_from_blob(body));
|
||||
let mut keywords = translation.keywords;
|
||||
let index = if mime_keywords {
|
||||
email_index_and_mime_keywords(body, &mut keywords)
|
||||
} else {
|
||||
email_index_from_blob(body)
|
||||
};
|
||||
let message_match = index_to_json(&index);
|
||||
let mailbox_ids = Value::Array(vec![Value::from(mailbox_local)]);
|
||||
let keywords = Value::Array(
|
||||
translation
|
||||
.keywords
|
||||
.iter()
|
||||
.map(|k| Value::String(k.clone()))
|
||||
.collect(),
|
||||
);
|
||||
let keywords = Value::Array(keywords.into_iter().map(Value::String).collect());
|
||||
tx.execute(
|
||||
"INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
@@ -1188,6 +1195,7 @@ fn refresh_present_flags(
|
||||
fetch_batch,
|
||||
fetch_batch_bytes: _,
|
||||
include_deleted,
|
||||
mime_keywords,
|
||||
logger: _,
|
||||
} = opts;
|
||||
let mut updated: u64 = 0;
|
||||
@@ -1210,23 +1218,19 @@ fn refresh_present_flags(
|
||||
if translation.has_deleted_flag && !include_deleted {
|
||||
continue;
|
||||
}
|
||||
let Some((local_id, existing)) =
|
||||
load_email_keywords(&tx, source_id, folder, uidvalidity, uid)?
|
||||
let Some(stored) = load_email_keywords(&tx, source_id, folder, uidvalidity, uid)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if keyword_set_differs(&existing, &translation.keywords) {
|
||||
let json = Value::Array(
|
||||
translation
|
||||
.keywords
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(Value::String)
|
||||
.collect(),
|
||||
);
|
||||
let mut keywords = translation.keywords;
|
||||
if mime_keywords && let Some(blob) = db::blobs::blob_bytes(&tx, stored.blob_id)? {
|
||||
add_mime_keywords(&blob, &mut keywords);
|
||||
}
|
||||
if keyword_set_differs(&stored.keywords, &keywords) {
|
||||
let json = Value::Array(keywords.into_iter().map(Value::String).collect());
|
||||
tx.execute(
|
||||
"UPDATE emails SET keywords = ?1 WHERE id = ?2",
|
||||
params![json.to_string(), local_id],
|
||||
params![json.to_string(), stored.local_id],
|
||||
)?;
|
||||
updated += 1;
|
||||
}
|
||||
@@ -1236,24 +1240,34 @@ fn refresh_present_flags(
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
struct StoredKeywords {
|
||||
local_id: i64,
|
||||
blob_id: i64,
|
||||
keywords: Vec<String>,
|
||||
}
|
||||
|
||||
fn load_email_keywords(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
source_id: i64,
|
||||
folder: &str,
|
||||
uidvalidity: u32,
|
||||
uid: u32,
|
||||
) -> Result<Option<(i64, Vec<String>)>, Error> {
|
||||
) -> Result<Option<StoredKeywords>, Error> {
|
||||
let Some(local_id) = db::imap_ids::local_for_email(tx, source_id, folder, uidvalidity, uid)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let kw_json: String = tx.query_row(
|
||||
"SELECT keywords FROM emails WHERE id = ?1",
|
||||
let (blob_id, kw_json): (i64, String) = tx.query_row(
|
||||
"SELECT blob_id, keywords FROM emails WHERE id = ?1",
|
||||
params![local_id],
|
||||
|r| r.get(0),
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
let kws: Vec<String> = serde_json::from_str(&kw_json).unwrap_or_default();
|
||||
Ok(Some((local_id, kws)))
|
||||
let keywords = serde_json::from_str(&kw_json).unwrap_or_default();
|
||||
Ok(Some(StoredKeywords {
|
||||
local_id,
|
||||
blob_id,
|
||||
keywords,
|
||||
}))
|
||||
}
|
||||
|
||||
fn keyword_set_differs(a: &[String], b: &[String]) -> bool {
|
||||
|
||||
@@ -5,15 +5,19 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use crate::imap::automap;
|
||||
use crate::imap::{automap, name::decode_mailbox_name};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ResolvedFolder {
|
||||
pub name: String,
|
||||
pub decoded_name: String,
|
||||
pub leaf: String,
|
||||
pub parent_path: Option<String>,
|
||||
pub path: PathBuf,
|
||||
@@ -57,6 +61,7 @@ pub fn discover(root: &Path, automap_enabled: bool) -> Result<Vec<ResolvedFolder
|
||||
}
|
||||
let mut folders = vec![ResolvedFolder {
|
||||
name: "INBOX".to_owned(),
|
||||
decoded_name: "INBOX".to_owned(),
|
||||
leaf: "INBOX".to_owned(),
|
||||
parent_path: None,
|
||||
path: root.to_path_buf(),
|
||||
@@ -97,15 +102,16 @@ pub fn discover(root: &Path, automap_enabled: bool) -> Result<Vec<ResolvedFolder
|
||||
if let Some(p) = parent_path.as_deref() {
|
||||
ensure_ephemeral_parents(p, &mut known, &mut folders, root);
|
||||
}
|
||||
let leaf = leaf_of(&canonical);
|
||||
let decoded_name = decode_name(&canonical);
|
||||
let role = if automap_enabled {
|
||||
automap::role_for_folder(&canonical, &[], "", true)
|
||||
automap::role_for_folder(&decoded_name, &[], "", true)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
folders.push(ResolvedFolder {
|
||||
leaf: decode_segment(leaf_of(&canonical)).into_owned(),
|
||||
name: canonical.clone(),
|
||||
leaf: display_leaf(leaf),
|
||||
decoded_name,
|
||||
parent_path,
|
||||
path,
|
||||
role,
|
||||
@@ -126,11 +132,11 @@ fn ensure_ephemeral_parents(
|
||||
if known.iter().any(|n| n == &ancestor) {
|
||||
continue;
|
||||
}
|
||||
let leaf = display_leaf(leaf_of(&ancestor));
|
||||
let parent_path = split_parent(&ancestor);
|
||||
folders.push(ResolvedFolder {
|
||||
decoded_name: decode_name(&ancestor),
|
||||
leaf: decode_segment(leaf_of(&ancestor)).into_owned(),
|
||||
name: ancestor.clone(),
|
||||
leaf,
|
||||
parent_path,
|
||||
path: root.to_path_buf(),
|
||||
role: None,
|
||||
@@ -157,11 +163,22 @@ fn split_parent(name: &str) -> Option<String> {
|
||||
name.rfind('.').map(|i| name[..i].to_owned())
|
||||
}
|
||||
|
||||
/// A folder's name as shown: Maildir++ keeps names in modified UTF-7, as
|
||||
/// Dovecot and Courier write them. A name that isn't valid modified UTF-7 is
|
||||
/// shown as it is on disk.
|
||||
fn display_leaf(raw: &str) -> String {
|
||||
crate::imap::name::decode_mailbox_name(raw).unwrap_or_else(|_| raw.to_owned())
|
||||
fn decode_segment(segment: &str) -> Cow<'_, str> {
|
||||
if !segment.contains('&') {
|
||||
return Cow::Borrowed(segment);
|
||||
}
|
||||
decode_mailbox_name(segment).map_or(Cow::Borrowed(segment), Cow::Owned)
|
||||
}
|
||||
|
||||
fn decode_name(name: &str) -> String {
|
||||
let mut decoded = String::with_capacity(name.len());
|
||||
for (i, segment) in name.split('.').enumerate() {
|
||||
if i > 0 {
|
||||
decoded.push('.');
|
||||
}
|
||||
decoded.push_str(&decode_segment(segment));
|
||||
}
|
||||
decoded
|
||||
}
|
||||
|
||||
fn leaf_of(name: &str) -> &str {
|
||||
@@ -174,7 +191,7 @@ fn leaf_of(name: &str) -> &str {
|
||||
pub fn apply_filters(folders: Vec<ResolvedFolder>, filters: &FolderFilters) -> Vec<ResolvedFolder> {
|
||||
folders
|
||||
.into_iter()
|
||||
.filter(|f| match_filters(&f.name, filters))
|
||||
.filter(|f| match_filters(&f.decoded_name, filters))
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -187,11 +204,11 @@ pub fn restore_ephemeral_parents(folders: &mut Vec<ResolvedFolder>, root: &Path)
|
||||
if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) {
|
||||
continue;
|
||||
}
|
||||
let leaf = display_leaf(leaf_of(&ancestor));
|
||||
let parent_path = split_parent(&ancestor);
|
||||
to_add.push(ResolvedFolder {
|
||||
decoded_name: decode_name(&ancestor),
|
||||
leaf: decode_segment(leaf_of(&ancestor)).into_owned(),
|
||||
name: ancestor,
|
||||
leaf,
|
||||
parent_path,
|
||||
path: root.to_path_buf(),
|
||||
role: None,
|
||||
@@ -386,9 +403,9 @@ mod tests {
|
||||
fn leaf_and_split_handle_root_and_nested() {
|
||||
assert_eq!(leaf_of("INBOX"), "INBOX");
|
||||
assert_eq!(leaf_of("Archive.2025"), "2025");
|
||||
assert_eq!(display_leaf("&AMk-quipe &- co"), "Équipe & co");
|
||||
assert_eq!(display_leaf("Plain"), "Plain");
|
||||
assert_eq!(display_leaf("Équipe"), "Équipe");
|
||||
assert_eq!(decode_segment("&AMk-quipe &- co"), "Équipe & co");
|
||||
assert_eq!(decode_segment("Plain"), "Plain");
|
||||
assert_eq!(decode_segment("Équipe"), "Équipe");
|
||||
assert_eq!(split_parent("INBOX"), None);
|
||||
assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned()));
|
||||
}
|
||||
@@ -398,6 +415,7 @@ mod tests {
|
||||
.iter()
|
||||
.map(|n| ResolvedFolder {
|
||||
name: (*n).to_owned(),
|
||||
decoded_name: (*n).to_owned(),
|
||||
leaf: (*n).to_owned(),
|
||||
parent_path: None,
|
||||
path: PathBuf::new(),
|
||||
@@ -502,4 +520,61 @@ mod tests {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// Dovecot stores non-ASCII and '&' in folder names as modified UTF-7.
|
||||
#[test]
|
||||
fn discover_decodes_modified_utf7_names() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
make_maildir(
|
||||
td.path(),
|
||||
&[".&AOk-t&AOk-", ".R&-D", ".Plain", ".&ZeVnLIqe-.&U,BTFw-"],
|
||||
);
|
||||
let folders = discover(td.path(), true).unwrap();
|
||||
let by_name = |name: &str| folders.iter().find(|f| f.name == name).unwrap();
|
||||
|
||||
let ete = by_name("&AOk-t&AOk-");
|
||||
assert_eq!(ete.decoded_name, "été");
|
||||
assert_eq!(ete.leaf, "été");
|
||||
assert_eq!(by_name("R&-D").leaf, "R&D");
|
||||
assert_eq!(by_name("Plain").leaf, "Plain");
|
||||
|
||||
let nested = by_name("&ZeVnLIqe-.&U,BTFw-");
|
||||
assert_eq!(nested.decoded_name, "日本語.台北");
|
||||
assert_eq!(nested.leaf, "台北");
|
||||
assert_eq!(nested.parent_path.as_deref(), Some("&ZeVnLIqe-"));
|
||||
let parent = by_name("&ZeVnLIqe-");
|
||||
assert!(parent.ephemeral);
|
||||
assert_eq!(parent.leaf, "日本語");
|
||||
}
|
||||
|
||||
// Names that are not valid modified UTF-7 are kept byte for byte.
|
||||
#[test]
|
||||
fn decode_keeps_literal_ampersands_and_utf8() {
|
||||
for name in ["R&D", "AT&T", "Tom & Jerry-2", "Été", "Sales&", "A&-B&C"] {
|
||||
assert_eq!(decode_name(name), name);
|
||||
}
|
||||
assert_eq!(decode_name("R&D.&AOk-t&AOk-"), "R&D.été");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filters_match_decoded_names() {
|
||||
let mut folders = folders_for(&["INBOX", "Plain"]);
|
||||
folders.push(ResolvedFolder {
|
||||
name: "&AOk-t&AOk-".to_owned(),
|
||||
decoded_name: "été".to_owned(),
|
||||
leaf: "été".to_owned(),
|
||||
parent_path: None,
|
||||
path: PathBuf::new(),
|
||||
role: None,
|
||||
ephemeral: false,
|
||||
});
|
||||
let filters = FolderFilters {
|
||||
include: Vec::new(),
|
||||
exclude: Vec::new(),
|
||||
explicit: vec!["été".to_owned()],
|
||||
};
|
||||
let kept = apply_filters(folders, &filters);
|
||||
assert_eq!(kept.len(), 1);
|
||||
assert_eq!(kept[0].name, "&AOk-t&AOk-");
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig {
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 2,
|
||||
|
||||
@@ -42,6 +42,7 @@ fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportCon
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 2,
|
||||
|
||||
@@ -183,6 +183,7 @@ fn dovecot_export_imap_round_trip_converges() {
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 1,
|
||||
|
||||
@@ -1009,3 +1009,76 @@ fn get_folder_messages_preserve_position_when_one_errors() {
|
||||
assert!(!msgs[1].success);
|
||||
assert!(msgs[2].success);
|
||||
}
|
||||
|
||||
fn plan_folders_against_mock(
|
||||
kind: inbuxa_migrate::exchange_ews::types::MailboxKind,
|
||||
mailbox: Option<&str>,
|
||||
body_matcher: Matcher,
|
||||
) {
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::import_exchange_ews::folders::plan_folders;
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
let url = format!("{}/EWS/Exchange.asmx", server.url());
|
||||
let find_folder = envelope(&format!(
|
||||
"<m:FindFolderResponse{NS}><m:ResponseMessages><m:FindFolderResponseMessage ResponseClass=\"Success\">\
|
||||
<m:ResponseCode>NoError</m:ResponseCode>\
|
||||
<m:RootFolder TotalItemsInView=\"0\" IncludesLastItemInRange=\"true\"><t:Folders/></m:RootFolder>\
|
||||
</m:FindFolderResponseMessage></m:ResponseMessages></m:FindFolderResponse>"
|
||||
));
|
||||
let get_folder = envelope(&format!(
|
||||
"<m:GetFolderResponse{NS}><m:ResponseMessages>\
|
||||
<m:GetFolderResponseMessage ResponseClass=\"Success\">\
|
||||
<m:ResponseCode>NoError</m:ResponseCode>\
|
||||
<m:Folders><t:Folder><t:FolderId Id=\"ROOT\" ChangeKey=\"K\"/></t:Folder></m:Folders>\
|
||||
</m:GetFolderResponseMessage></m:ResponseMessages></m:GetFolderResponse>"
|
||||
));
|
||||
let find = server
|
||||
.mock("POST", "/EWS/Exchange.asmx")
|
||||
.match_header("soapaction", Matcher::Regex("/FindFolder\"$".into()))
|
||||
.match_body(body_matcher.clone())
|
||||
.with_status(200)
|
||||
.with_header("content-type", TXT_XML)
|
||||
.with_body(find_folder)
|
||||
.expect(1)
|
||||
.create();
|
||||
let get = server
|
||||
.mock("POST", "/EWS/Exchange.asmx")
|
||||
.match_header("soapaction", Matcher::Regex("/GetFolder\"$".into()))
|
||||
.match_body(body_matcher)
|
||||
.with_status(200)
|
||||
.with_header("content-type", TXT_XML)
|
||||
.with_body(get_folder)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let plan = plan_folders(&client(0), &url, kind, mailbox, Logger::new(0))
|
||||
.expect("folder planning should succeed");
|
||||
assert_eq!(plan.root_folder_id, "ROOT");
|
||||
find.assert();
|
||||
get.assert();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_mailbox_addresses_distinguished_folders_in_that_mailbox() {
|
||||
// Regression for importing a FullAccess mailbox with Basic auth: folders must name --mailbox.
|
||||
plan_folders_against_mock(
|
||||
inbuxa_migrate::exchange_ews::types::MailboxKind::Primary,
|
||||
Some("[email protected]"),
|
||||
Matcher::Regex(
|
||||
"<t:DistinguishedFolderId Id=\"[a-z]+\"><t:Mailbox><t:EmailAddress>[email protected]</t:EmailAddress></t:Mailbox></t:DistinguishedFolderId>".into(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_folders_ignore_explicit_mailbox() {
|
||||
// Public folder roots cannot carry a Mailbox element.
|
||||
plan_folders_against_mock(
|
||||
inbuxa_migrate::exchange_ews::types::MailboxKind::PublicFolders,
|
||||
Some("[email protected]"),
|
||||
Matcher::Regex(
|
||||
"^(?s)[^@]*<t:DistinguishedFolderId Id=\"publicfoldersroot\"/>[^@]*$".into(),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -170,6 +170,7 @@ fn run_import(
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 256,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 1,
|
||||
@@ -922,6 +923,71 @@ fn coordinator_present_flag_change_updates_keywords() {
|
||||
);
|
||||
}
|
||||
|
||||
const MSG_BODY_KEYWORDS: &[u8] = b"From: a@b\r\nTo: c@d\r\nSubject: hi\r\nMessage-ID: <m2@h>\r\nKeywords: bearbeitet von KL, VIP\r\nDate: Mon, 12 May 2025 10:00:00 +0000\r\n\r\nhello";
|
||||
|
||||
fn stored_keywords(archive: &PathBuf) -> Vec<String> {
|
||||
let dbc = Connection::open(archive).unwrap();
|
||||
let kw: String = dbc
|
||||
.query_row("SELECT keywords FROM emails LIMIT 1", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
let mut kws: Vec<String> = serde_json::from_str(&kw).unwrap();
|
||||
kws.sort_unstable();
|
||||
kws
|
||||
}
|
||||
|
||||
// With --mime-keywords the Keywords header is imported and survives a later IMAP flag refresh.
|
||||
#[test]
|
||||
fn coordinator_mime_keywords_imported_and_kept_on_flag_refresh() {
|
||||
let mut scripts: Vec<Script> = Vec::new();
|
||||
scripts.extend(single_inbox_scripts(100, 2, MSG_BODY_KEYWORDS));
|
||||
scripts.push(control_script_present_flags(
|
||||
100,
|
||||
2,
|
||||
&[1],
|
||||
&[(1, "\\Seen \\Flagged")],
|
||||
));
|
||||
scripts.push(worker_idle_script("IMAP4rev2 LITERAL+ AUTH=PLAIN"));
|
||||
let server = MockImap::start_scripts(scripts);
|
||||
let archive = tempfile("mime_keywords");
|
||||
run_import(&server, "alice", archive.clone(), |c| {
|
||||
c.mime_keywords = true
|
||||
})
|
||||
.expect("first import");
|
||||
let first = stored_keywords(&archive);
|
||||
assert!(
|
||||
first.contains(&"bearbeitet_von_kl".to_owned()) && first.contains(&"vip".to_owned()),
|
||||
"{first:?}"
|
||||
);
|
||||
let summary = run_import(&server, "alice", archive.clone(), |c| {
|
||||
c.mime_keywords = true
|
||||
})
|
||||
.expect("second import");
|
||||
let email = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(k, _)| *k == "email")
|
||||
.unwrap();
|
||||
assert_eq!(email.1.created, 0);
|
||||
assert_eq!(email.1.updated, 1);
|
||||
let mut expected = first;
|
||||
expected.push("$flagged".to_owned());
|
||||
expected.sort_unstable();
|
||||
assert_eq!(stored_keywords(&archive), expected);
|
||||
}
|
||||
|
||||
// Without --mime-keywords the Keywords header is ignored.
|
||||
#[test]
|
||||
fn coordinator_ignores_mime_keywords_by_default() {
|
||||
let server = MockImap::start_scripts(single_inbox_scripts(100, 2, MSG_BODY_KEYWORDS));
|
||||
let archive = tempfile("mime_keywords_off");
|
||||
run_import(&server, "alice", archive.clone(), |_| {}).expect("import");
|
||||
let kws = stored_keywords(&archive);
|
||||
assert!(
|
||||
!kws.iter().any(|k| k == "vip" || k == "bearbeitet_von_kl"),
|
||||
"{kws:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_present_newly_deleted_is_left_intact() {
|
||||
let mut scripts: Vec<Script> = Vec::new();
|
||||
@@ -2625,6 +2691,7 @@ fn dry_run_never_selects_noselect_ancestors() {
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 256,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 1,
|
||||
|
||||
@@ -74,6 +74,7 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig {
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 256,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 4,
|
||||
@@ -97,6 +98,7 @@ fn dovecot_config(account: &Account, imap: &integration::Endpoint) -> ImapImport
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
mime_keywords: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 2,
|
||||
|
||||
Reference in new issue
Block a user