From 50942574eaf9dc0e724dc0ad72393221ccd143e7 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 29 Oct 2026 23:14:41 -0700 Subject: [PATCH] Add export-imap: an archive's mail into any IMAP server For a target that speaks neither JMAP nor takes files. Like export it keeps no state: each run reads the target folder (UID, flags, size and the match headers), appends what's missing with its flags and arrival time, and brings flags up to date on what's there, so a second run writes nothing and an interrupted one is finished by running it again. Folders follow the server: INBOX, the SPECIAL-USE folder for sent, drafts, trash, junk and archive (created with the mark where allowed), everything else under the personal namespace with the server's separator. Keywords go where PERMANENTFLAGS allows; $deleted never does. The IMAP client gains APPEND (LITERAL+ or the go-ahead, with a NO before it ending the command cleanly), CREATE, SUBSCRIBE and UID STORE, and reads a FETCH section with a header list as one item. tests/integration_export_imap.rs (live Dovecot, ignored like the other container tests) round-trips through import imap and checks the second and third runs; tests/mock_imap.rs covers both literal paths and the early NO. --- CHANGELOG.md | 7 + README.md | 12 +- docs/usage.md | 45 +- src/cli.rs | 75 +++ src/export_files/mail.rs | 2 +- src/imap/client.rs | 82 +++ src/imap/command.rs | 46 ++ src/imap/response.rs | 28 +- src/main.rs | 7 + src/sync/export_imap.rs | 787 ++++++++++++++++++++++++++++ src/sync/import_imap/coordinator.rs | 6 +- src/sync/mod.rs | 1 + tests/integration_export_imap.rs | 223 ++++++++ tests/mock_imap.rs | 87 +++ 14 files changed, 1395 insertions(+), 13 deletions(-) create mode 100644 src/sync/export_imap.rs create mode 100644 tests/integration_export_imap.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b47d2..1cdf381 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,18 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a `.ics`, Sieve scripts, the file tree, and a `manifest.json` with what those formats don't hold. It needs no network, writes only into a new or empty directory, and keeps the directory private to its owner. +- `export-imap` writes an archive's mail into any IMAP server: folders in the + server's own layout (SPECIAL-USE folders for sent, drafts, trash, junk and + archive), flags, keywords where the folder keeps them, and arrival times. + Like `export`, it matches what is already there, so running it again + writes only what changed. ### Changed - `import maildir` reads folder names in modified UTF-7, as Dovecot and Courier write them, and the custom keywords in each folder's `dovecot-keywords` file (Junk, labels). +- A FETCH section with a header list, such as + `BODY[HEADER.FIELDS (MESSAGE-ID)]`, is read as one item. - `import takeout` unfolds and unescapes a calendar's `X-WR-CALNAME` and `X-WR-CALDESC`, so a long name or one with a comma reads back whole. diff --git a/README.md b/README.md index a5e5ed7..af0c45f 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate inspect alice.sqlite what landed inbuxa-migrate export … alice.sqlite the archive into inbuxa inbuxa-migrate export-files alice.sqlite out the archive as plain files + inbuxa-migrate export-imap … alice.sqlite its mail into any IMAP server Between the two sides sits an archive: one SQLite file holding one account. `import` fills it from the old server; `export` writes it into the new one. @@ -34,11 +35,12 @@ restore it later with `export`. An archive remembers which account filled it and refuses a different one unless told otherwise. Export speaks JMAP only. It writes into inbuxa, or into any JMAP server that -advertises the types being written. `export-files` is the way out to -everything else: it writes the archive as a Maildir++ tree, vCard, -iCalendar and Sieve files that other servers and mail, calendar and -contacts apps open, with no network and no store format in the way. The full -command reference is in [docs/usage.md](docs/usage.md). +advertises the types being written. `export-imap` writes the mail into any +IMAP server, and `export-files` is the way out to everything else: it writes +the archive as a Maildir++ tree, vCard, iCalendar and Sieve files that other +servers and mail, calendar and contacts apps open, with no network and no +store format in the way. The full command reference is in +[docs/usage.md](docs/usage.md). ## Sources diff --git a/docs/usage.md b/docs/usage.md index d7820aa..30c510c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,7 +13,7 @@ and how to install it. ## Commands and global flags ``` -inbuxa-migrate [args...] +inbuxa-migrate [args...] ``` Every command takes a few global flags -- verbosity, worker pool size, retry @@ -260,7 +260,8 @@ failed. `--prune` also deletes what is on the target and not in the archive. It asks first; `--yes` answers for it, for scripts. Export speaks JMAP only; for -anything else, see [Export to files](#export-to-files). +other servers, see [Export to IMAP](#export-to-imap) and +[Export to files](#export-to-files). With `--dry-run`, export reads the target and writes nothing to the account. It prints the plan in plain words: for each type, how much would be @@ -281,6 +282,46 @@ The plan lists each predicted failure and its reason. When anything would fail, the dry run exits 5, as the real run would, so a script can stop before it starts. +## Export to IMAP + +``` +inbuxa-migrate export-imap \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ + [--allow-cleartext] [--dry-run] \ + +``` + +Writes an archive's mail into an account on any IMAP server, for a target +that doesn't speak JMAP. It signs in the way `import imap` does, with the +same flags. Contacts, calendars and filters don't go over IMAP; write those +with `export-files`. + +Like `export`, it keeps no state. Each run reads the target folder first, +matching messages by Message-ID, or without one by sender, subject, date and +recipients, with size deciding between equals. What's missing is appended, +with its flags and the time it arrived; what's there gets the archive's +flags if they differ. So a second run writes nothing, and an interrupted run +is finished by running it again: a message that landed before the +connection dropped is found, not doubled. + +- **Folders.** The inbox goes to INBOX. A sent, drafts, trash, junk or + archive folder goes to the one the server marks for it with SPECIAL-USE, + such as Gmail's `[Gmail]/Sent Mail`, and is created with that mark when the + server allows it. Every other folder is created under the account's + personal namespace with its path, using the server's separator; that + separator inside a name becomes `_`. Names go as UTF-8 to a server that + accepts it, in modified UTF-7 otherwise. +- **Flags.** Read, answered, flagged and draft are IMAP's own. Other keywords + (Junk, labels) go along where the folder keeps keywords, and the run says + how many were left off where it doesn't. Deleted isn't carried, so the + next client to expunge removes nothing that was written. +- A message the server refuses (too large, over quota) is counted as failed + and the run goes on; a dropped connection ends the run. + +With `--dry-run` it signs in, reads the target with `EXAMINE`, and counts +what it would create, append and update, without writing anything. + ## Export to files ``` diff --git a/src/cli.rs b/src/cli.rs index 2a832ef..032a02d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -20,6 +20,7 @@ use crate::jmap::account::AccountSelector; use crate::jmap::http::Auth; use crate::logging::Logger; use crate::secret; +use crate::sync::export_imap::ExportImapConfig; use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig}; use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; @@ -55,6 +56,11 @@ enum Command { name = "export-files" )] ExportFiles(Box), + #[command( + about = "Write a local archive's mail into an IMAP server", + name = "export-imap" + )] + ExportImap(Box), #[command(about = "Show the contents of a local archive (read-only)")] Inspect(Box), } @@ -365,6 +371,55 @@ struct ExportArgs { archive: PathBuf, } +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +struct ExportImapArgs { + #[arg( + long, + value_name = "URL", + help = "imap://host[:port] or imaps://host[:port]" + )] + url: String, + + #[arg( + long, + value_name = "USER", + help = "PLAIN (preferred) with LOGIN fallback" + )] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token for XOAUTH2 (value optional: falls back to $INBUXA_MIGRATE_API_KEY or prompt)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "USER", + help = "XOAUTH2 identity (required with --auth-bearer)" + )] + auth_user: Option, + + #[arg(long, help = "Permit credentials on imap:// without STARTTLS")] + allow_cleartext: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, +} + #[derive(Args)] struct ExportFilesArgs { #[arg( @@ -428,6 +483,7 @@ pub enum Action { ImportExchangeGraph(CommonConfig, GraphImportConfig), Export(CommonConfig, ExportConfig), ExportFiles(ExportFilesConfig), + ExportImap(CommonConfig, ExportImapConfig), Inspect(InspectConfig), } @@ -554,6 +610,7 @@ impl Cli { Command::Import { source } => resolve_import(source), Command::Export(args) => resolve_export(*args), Command::ExportFiles(args) => resolve_export_files(*args), + Command::ExportImap(args) => resolve_export_imap(*args), Command::Inspect(args) => resolve_inspect(*args), } } @@ -932,6 +989,24 @@ fn resolve_export(args: ExportArgs) -> Result { )) } +fn resolve_export_imap(args: ExportImapArgs) -> Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_imap_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + args.auth_user.as_deref(), + )?; + Ok(Action::ExportImap( + common, + ExportImapConfig { + url: args.url, + auth, + allow_cleartext: args.allow_cleartext, + }, + )) +} + fn resolve_export_files(args: ExportFilesArgs) -> Result { let objects = args.objects.as_deref().map(parse_object_list).transpose()?; Ok(Action::ExportFiles(ExportFilesConfig { diff --git a/src/export_files/mail.rs b/src/export_files/mail.rs index 46a4f9c..79fd231 100644 --- a/src/export_files/mail.rs +++ b/src/export_files/mail.rs @@ -312,7 +312,7 @@ fn flags_for(keywords: &[String], folder: &mut Folder, logger: Logger) -> String /// When the message arrived, as Unix seconds: the archive's received time, /// else its Date header, else now. -fn delivery_secs(received_at: &str, message: &[u8]) -> i64 { +pub(crate) fn delivery_secs(received_at: &str, message: &[u8]) -> i64 { let parsed = OffsetDateTime::parse(received_at, &Rfc3339) .or_else(|_| OffsetDateTime::parse(received_at, &Rfc2822)) .map(|t| t.unix_timestamp()) diff --git a/src/imap/client.rs b/src/imap/client.rs index 066ce26..2ad198a 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -443,6 +444,87 @@ impl ImapClient { } } + /// APPEND one message. With LITERAL+ the message goes straight after the + /// command; otherwise it waits for the server's go-ahead, and a NO + /// before that (too large, over quota) ends the command without sending + /// it. + pub fn append( + &mut self, + mailbox: &str, + flags: &[String], + internal_date: Option<&str>, + message: &[u8], + ) -> Result { + let head = command::append_head(mailbox, flags, internal_date); + if command::contains_literal(head.as_bytes()) + && !self.has_capability("LITERAL+") + && !self.has_capability("LITERAL-") + { + return Err(ImapError::Unsupported( + "mailbox name needs a literal but server does not advertise LITERAL+ or LITERAL-" + .into(), + )); + } + let non_sync = self.has_capability("LITERAL+"); + let tag = self.tags.next_tag(); + let started = Instant::now(); + let mut out = format!( + "{tag} {head} {{{}{}}}\r\n", + message.len(), + if non_sync { "+" } else { "" } + ) + .into_bytes(); + if non_sync { + out.extend_from_slice(message); + out.extend_from_slice(b"\r\n"); + } + self.write_all(&out)?; + let mut sent = non_sync; + loop { + match parse_response(&mut self.reader)? { + Response::Continuation(_) if !sent => { + let mut body = Vec::with_capacity(message.len() + 2); + body.extend_from_slice(message); + body.extend_from_slice(b"\r\n"); + self.write_all(&body)?; + sent = true; + } + Response::Continuation(_) => { + return Err(ImapError::Protocol( + "unexpected second continuation during APPEND".into(), + )); + } + Response::Tagged { tag: t, line } if t == tag => { + self.logger.trace_cmd( + "IMAP", + &head, + &format!("{} {}", status_word(&line.status), line.text), + started.elapsed(), + ); + return match line.status { + Status::Ok => Ok(line), + Status::No => Err(ImapError::No(line.into_no_error())), + Status::Bad => Err(ImapError::Bad(line.text)), + Status::Bye => Err(ImapError::Bye(line.text)), + Status::PreAuth => Err(ImapError::Protocol( + "unexpected PREAUTH on tagged response".into(), + )), + }; + } + Response::Tagged { tag: other, .. } => { + return Err(ImapError::Protocol(format!( + "expected tag {tag}, got {other}" + ))); + } + Response::Untagged(Untagged::Bye(text)) => { + self.closed = true; + return Err(ImapError::Bye(text)); + } + Response::Untagged(_) => {} + } + } + } + fn finish_tagged(&mut self, line: StatusLine) -> Result<(), ImapError> { match line.status { Status::Ok => Ok(()), diff --git a/src/imap/command.rs b/src/imap/command.rs index 611bdd8..fc77264 100644 --- a/src/imap/command.rs +++ b/src/imap/command.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -205,6 +206,31 @@ pub fn select(mailbox: &str) -> String { format!("SELECT {}", quote_astring(mailbox)) } +/// `APPEND mailbox (flags) "date"`, without the literal that follows. +pub fn append_head(mailbox: &str, flags: &[String], internal_date: Option<&str>) -> String { + let mut out = format!("APPEND {} ({})", quote_astring(mailbox), flags.join(" ")); + if let Some(date) = internal_date { + out.push_str(&format!(" \"{date}\"")); + } + out +} + +pub fn create(mailbox: &str, special_use: Option<&str>) -> String { + match special_use { + Some(attr) => format!("CREATE {} (USE ({attr}))", quote_astring(mailbox)), + None => format!("CREATE {}", quote_astring(mailbox)), + } +} + +pub fn subscribe(mailbox: &str) -> String { + format!("SUBSCRIBE {}", quote_astring(mailbox)) +} + +/// Replace a message's flags, without the untagged FETCH echo. +pub fn uid_store_flags(uid: u32, flags: &[String]) -> String { + format!("UID STORE {uid} FLAGS.SILENT ({})", flags.join(" ")) +} + pub fn examine(mailbox: &str) -> String { format!("EXAMINE {}", quote_astring(mailbox)) } @@ -333,6 +359,26 @@ mod tests { assert_eq!(quote_astring("INBOX"), "\"INBOX\""); } + #[test] + fn write_commands_are_shaped_as_rfc_9051_says() { + let flags = vec!["\\Seen".to_owned(), "$Junk".to_owned()]; + assert_eq!( + append_head("Archive/2024", &flags, Some(" 1-Mar-2024 12:00:00 +0000")), + "APPEND \"Archive/2024\" (\\Seen $Junk) \" 1-Mar-2024 12:00:00 +0000\"" + ); + assert_eq!(append_head("INBOX", &[], None), "APPEND \"INBOX\" ()"); + assert_eq!( + create("Sent", Some("\\Sent")), + "CREATE \"Sent\" (USE (\\Sent))" + ); + assert_eq!(create("a", None), "CREATE \"a\""); + assert_eq!(subscribe("a"), "SUBSCRIBE \"a\""); + assert_eq!( + uid_store_flags(7, &flags), + "UID STORE 7 FLAGS.SILENT (\\Seen $Junk)" + ); + } + #[test] fn quote_astring_escapes_quote_and_backslash() { assert_eq!(quote_astring("a\"b"), "\"a\\\"b\""); diff --git a/src/imap/response.rs b/src/imap/response.rs index 1bbc097..7877f09 100644 --- a/src/imap/response.rs +++ b/src/imap/response.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -420,11 +421,18 @@ impl<'r, R: BufRead> Parser<'r, R> { fn read_fetch_item_name(&mut self) -> Result { let start = self.pos; + // A section such as `BODY[HEADER.FIELDS (MESSAGE-ID)]` is one name, + // spaces and parentheses included. + let mut depth = 0usize; while let Some(b) = self.peek() { match b { - b' ' | b'\r' | b'\n' | b')' | b'(' => break, - _ => self.bump(), + b'[' => depth += 1, + b']' => depth = depth.saturating_sub(1), + b'\r' | b'\n' => break, + b' ' | b')' | b'(' if depth == 0 => break, + _ => {} } + self.bump(); } let bytes = &self.buf[start..self.pos]; std::str::from_utf8(bytes) @@ -1054,6 +1062,22 @@ mod tests { } } + #[test] + fn fetch_section_with_a_header_list_is_one_item() { + let input = b"* 3 FETCH (UID 9 BODY[HEADER.FIELDS (MESSAGE-ID DATE)] {22}\r\nMessage-ID: \r\n\r\n FLAGS (\\Seen))\r\n"; + let mut reader = std::io::BufReader::new(&input[..]); + match parse_response(&mut reader).unwrap() { + Response::Untagged(Untagged::Fetch { items, .. }) => { + let names: Vec<&str> = items.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + names, + ["UID", "BODY[HEADER.FIELDS (MESSAGE-ID DATE)]", "FLAGS"] + ); + } + other => panic!("unexpected {other:?}"), + } + } + #[test] fn untagged_fetch_with_body_literal() { let input = b"* 1 FETCH (UID 5 BODY[] {11}\r\nHello world)\r\n"; diff --git a/src/main.rs b/src/main.rs index a45972a..735945b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,6 +87,13 @@ fn run() -> i32 { let logger = config.logger; (RunOutcome::from_result(export_files::run(config)), logger) } + Action::ExportImap(common, config) => { + let logger = common.logger; + ( + RunOutcome::from_result(sync::export_imap::run(common, config)), + logger, + ) + } Action::Inspect(config) => { return match inspect::run(config) { Ok(()) => 0, diff --git a/src/sync/export_imap.rs b/src/sync/export_imap.rs new file mode 100644 index 0000000..4294b30 --- /dev/null +++ b/src/sync/export_imap.rs @@ -0,0 +1,787 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-imap`: an archive's mail written into any IMAP server, for a +//! target that speaks neither JMAP nor takes files. Like `export`, it keeps +//! no state: every run reads the target first, appends only what isn't +//! there and brings flags up to date on what is, so an interrupted run is +//! finished by running it again. + +use std::collections::{BTreeSet, HashMap}; + +use encodify::utf7; +use rusqlite::Connection; +use time::OffsetDateTime; +use time::macros::format_description; + +use crate::db::{blobs, init}; +use crate::error::Error; +use crate::export_files::mail::delivery_secs; +use crate::export_files::paths::Names; +use crate::imap::client::{ConnectMode, ImapClient}; +use crate::imap::command; +use crate::imap::error::ImapError; +use crate::imap::name::decode_mailbox_name_with; +use crate::imap::response::{Untagged, Value}; +use crate::imap::transport::Connector; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::emailmeta::email_index_from_blob; +use crate::sync::import_imap::ImapAuth; +use crate::sync::import_imap::coordinator::{authenticate, discover_namespace, parse_endpoint}; +use crate::sync::keys::EmailIndex; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +pub struct ExportImapConfig { + pub url: String, + pub auth: ImapAuth, + pub allow_cleartext: bool, +} + +/// The header fields matching reads from what the target already holds: +/// the same ones the archive's messages are matched by. +const MATCH_HEADERS: &str = "BODY.PEEK[HEADER.FIELDS (MESSAGE-ID FROM TO SUBJECT DATE)]"; + +/// Archive roles and the SPECIAL-USE attribute a target marks the same +/// folder with. +const SPECIAL_USE: [(&str, &str); 5] = [ + ("sent", "\\Sent"), + ("drafts", "\\Drafts"), + ("trash", "\\Trash"), + ("junk", "\\Junk"), + ("archive", "\\Archive"), +]; + +pub fn run(common: CommonConfig, config: ExportImapConfig) -> Result { + if !common.archive.is_file() { + return Err(Error::Usage(format!( + "no archive at {}", + common.archive.display() + ))); + } + let conn = init::open(&common.archive)?; + let logger = common.logger; + let dry_run = common.dry_run; + + let endpoint = parse_endpoint(&config.url)?; + let connector = + Connector::new(common.allow_invalid_certs).map_err(|e| Error::Connection(e.to_string()))?; + let mode = if endpoint.implicit_tls { + ConnectMode::ImplicitTls + } else if config.allow_cleartext { + ConnectMode::Plain + } else { + ConnectMode::StartTls + }; + let mut client = ImapClient::connect(&connector, &endpoint.host, endpoint.port, mode, logger) + .map_err(|e| Error::Connection(e.to_string()))?; + let user = authenticate(&mut client, &config.auth)?; + client + .refresh_capabilities() + .map_err(|e| Error::Connection(format!("post-auth CAPABILITY: {e}")))?; + if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { + let _ = client.enable(&["UTF8=ACCEPT"]); + } + let utf8 = client.utf8_accept(); + log_at( + logger, + &format!("signed in as {user} on {}:{}", endpoint.host, endpoint.port), + ); + + let prefix = discover_namespace(&mut client, logger)?; + let (delimiter, existing) = list_target(&mut client, utf8)?; + let mailboxes = read_mailboxes(&conn)?; + let mut folders = plan_folders(&mailboxes, &existing, &prefix, delimiter); + + let mut mailbox_counts = TypeCounts::default(); + let can_mark_use = client.has_capability("CREATE-SPECIAL-USE"); + let mut order: Vec<&mut Planned> = folders.values_mut().collect(); + order.sort_by(|a, b| a.name.cmp(&b.name)); + for folder in order { + if folder.exists { + mailbox_counts.skipped += 1; + continue; + } + if !dry_run { + let wire = wire_name(&folder.name, utf8); + let use_attr = folder.special_use.filter(|_| can_mark_use); + if let Err(e) = client.run_collect(&command::create(&wire, use_attr)) { + if use_attr.is_some() { + // A server can refuse the attribute and still take the name. + client + .run_collect(&command::create(&wire, None)) + .map_err(|e| folder_error(&folder.name, "CREATE", e))?; + } else { + return Err(folder_error(&folder.name, "CREATE", e)); + } + } + if folder.subscribed { + let _ = client.run_collect(&command::subscribe(&wire)); + } + } + folder.exists = !dry_run; + mailbox_counts.created += 1; + } + + let by_folder = read_emails(&conn)?; + let mut email_counts = TypeCounts::default(); + let mut names: Vec<(i64, String)> = folders + .iter() + .map(|(id, f)| (*id, f.name.clone())) + .collect(); + names.sort_by(|a, b| a.1.cmp(&b.1)); + for (mailbox_id, _) in names { + let folder = &folders[&mailbox_id]; + let Some(emails) = by_folder.get(&mailbox_id) else { + continue; + }; + sync_folder( + &mut client, + &conn, + folder, + emails, + utf8, + dry_run, + logger, + &mut email_counts, + )?; + } + let _ = client.logout(); + if dry_run { + logger.warn("dry run: nothing was written to the server"); + } + Ok(Summary { + per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], + ..Default::default() + }) +} + +struct Mailbox { + id: i64, + name: String, + parent_id: Option, + role: Option, + is_subscribed: bool, +} + +struct TargetFolder { + name: String, + attributes: Vec, +} + +#[derive(Debug)] +struct Planned { + name: String, + exists: bool, + special_use: Option<&'static str>, + subscribed: bool, +} + +struct ArchiveEmail { + id: i64, + blob_id: i64, + received_at: String, + keywords: Vec, +} + +fn log_at(logger: Logger, message: &str) { + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("{message}"); + } +} + +fn folder_error(name: &str, what: &str, e: ImapError) -> Error { + Error::Connection(format!("{what} {name:?}: {e}")) +} + +fn wire_name(name: &str, utf8: bool) -> String { + if utf8 { + name.to_owned() + } else { + utf7::IMAP.encode(name) + } +} + +/// The target's hierarchy separator and the folders it already has. +fn list_target(client: &mut ImapClient, utf8: bool) -> Result<(char, Vec), Error> { + let probe = client + .run_collect(&command::list("", "")) + .map_err(|e| Error::Connection(format!("LIST: {e}")))?; + let delimiter = probe + .untagged + .iter() + .find_map(|u| match u { + Untagged::List { delimiter, .. } => *delimiter, + _ => None, + }) + .unwrap_or('/'); + let all = client + .run_collect(&command::list("", "*")) + .map_err(|e| Error::Connection(format!("LIST: {e}")))?; + let folders = all + .untagged + .into_iter() + .filter_map(|u| match u { + Untagged::List { + attributes, name, .. + } => Some(TargetFolder { + name: canonical_inbox( + &decode_mailbox_name_with(&name, utf8).unwrap_or(name.clone()), + ), + attributes, + }), + _ => None, + }) + .collect(); + Ok((delimiter, folders)) +} + +fn canonical_inbox(name: &str) -> String { + if name.eq_ignore_ascii_case("INBOX") { + "INBOX".to_owned() + } else { + name.to_owned() + } +} + +fn read_mailboxes(conn: &Connection) -> Result, Error> { + let mut stmt = conn.prepare( + "SELECT id, name, parent_id, role, is_subscribed FROM mailboxes ORDER BY sort_order, id", + )?; + let rows = stmt.query_map([], |row| { + Ok(Mailbox { + id: row.get(0)?, + name: row.get(1)?, + parent_id: row.get(2)?, + role: row.get(3)?, + is_subscribed: row.get::<_, i64>(4)? != 0, + }) + })?; + Ok(rows.collect::>()?) +} + +/// Where each archive folder goes on the target: the inbox to INBOX, a +/// sent, drafts, trash, junk or archive folder to the one the target marks +/// for it, everything else under the personal namespace by its path. +fn plan_folders( + mailboxes: &[Mailbox], + existing: &[TargetFolder], + prefix: &str, + delimiter: char, +) -> HashMap { + let by_id: HashMap = mailboxes.iter().map(|m| (m.id, m)).collect(); + let depth = |m: &Mailbox| { + let mut d = 0; + let mut cur = m.parent_id; + while let Some(p) = cur { + d += 1; + if d > by_id.len() { + break; + } + cur = by_id.get(&p).and_then(|m| m.parent_id); + } + d + }; + let mut order: Vec<&Mailbox> = mailboxes.iter().collect(); + order.sort_by_key(|m| depth(m)); + + let mut names = Names::default(); + names.reserve("INBOX"); + let mut taken_use: BTreeSet<&str> = BTreeSet::new(); + let mut inbox_done = false; + let mut out: HashMap = HashMap::new(); + for mailbox in order { + let role = mailbox.role.as_deref(); + let special = SPECIAL_USE + .iter() + .find(|(r, _)| Some(*r) == role) + .map(|(_, a)| *a); + let name = if role == Some("inbox") && !inbox_done { + inbox_done = true; + "INBOX".to_owned() + } else if let Some(found) = special.filter(|a| !taken_use.contains(a)).and_then(|attr| { + existing + .iter() + .find(|f| f.attributes.iter().any(|x| x.eq_ignore_ascii_case(attr))) + }) { + taken_use.insert(special.expect("found by it")); + names.reserve(&found.name); + found.name.clone() + } else { + let segment = segment(&mailbox.name, delimiter); + let wanted = match mailbox + .parent_id + .filter(|p| *p != mailbox.id) + .and_then(|p| out.get(&p)) + { + Some(parent) => format!("{}{delimiter}{segment}", parent.name), + None => format!("{prefix}{segment}"), + }; + names.unique(&wanted, "") + }; + let exists = existing.iter().any(|f| f.name == name); + out.insert( + mailbox.id, + Planned { + name, + exists, + special_use: special, + subscribed: mailbox.is_subscribed, + }, + ); + } + out +} + +/// One level of a folder's name; the target's separator inside it would +/// split it in two. +fn segment(name: &str, delimiter: char) -> String { + let out: String = name + .chars() + .map(|c| if c == delimiter { '_' } else { c }) + .collect(); + if out.trim().is_empty() { + "_".to_owned() + } else { + out + } +} + +fn read_emails(conn: &Connection) -> Result>, Error> { + let mut stmt = conn.prepare( + "SELECT id, blob_id, received_at, mailbox_ids, keywords FROM emails ORDER BY id", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + })?; + let mut out: HashMap> = HashMap::new(); + for row in rows { + let (id, blob_id, received_at, mailbox_ids, keywords) = row?; + let keywords: Vec = serde_json::from_str(&keywords).unwrap_or_default(); + for mailbox in serde_json::from_str::>(&mailbox_ids).unwrap_or_default() { + out.entry(mailbox).or_default().push(ArchiveEmail { + id, + blob_id, + received_at: received_at.clone(), + keywords: keywords.clone(), + }); + } + } + Ok(out) +} + +struct OnTarget { + uid: u32, + flags: BTreeSet, + size: u64, + index: EmailIndex, + taken: bool, +} + +/// What the target lets a folder's messages keep: from PERMANENTFLAGS, or +/// everything when the server doesn't say. +struct Allowed { + any_keyword: bool, + listed: BTreeSet, +} + +#[allow(clippy::too_many_arguments)] +fn sync_folder( + client: &mut ImapClient, + conn: &Connection, + folder: &Planned, + emails: &[ArchiveEmail], + utf8: bool, + dry_run: bool, + logger: Logger, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let wire = wire_name(&folder.name, utf8); + let (mut present, allowed) = if folder.exists { + read_folder(client, &wire, dry_run).map_err(|e| folder_error(&folder.name, "SELECT", e))? + } else { + ( + Vec::new(), + Allowed { + any_keyword: true, + listed: BTreeSet::new(), + }, + ) + }; + let mut by_mid: HashMap> = HashMap::new(); + let mut by_fallback: HashMap<[u8; 32], Vec> = HashMap::new(); + for (i, t) in present.iter().enumerate() { + if t.index.mids.is_empty() { + by_fallback.entry(t.index.fb).or_default().push(i); + } + for mid in &t.index.mids { + by_mid.entry(mid.clone()).or_default().push(i); + } + } + + let (mut created, mut updated, mut unchanged, mut dropped) = (0u64, 0u64, 0u64, 0u64); + for email in emails { + let Some(bytes) = blobs::blob_bytes(conn, email.blob_id)? else { + logger.warn(&format!( + "email {}: its message is missing from the archive", + email.id + )); + counts.failed += 1; + continue; + }; + let index = email_index_from_blob(&bytes); + let (flags, left_off) = imap_flags(&email.keywords, &allowed); + dropped += left_off; + let candidates: Vec = if index.mids.is_empty() { + by_fallback.get(&index.fb).cloned().unwrap_or_default() + } else { + index + .mids + .iter() + .flat_map(|m| by_mid.get(m).cloned().unwrap_or_default()) + .collect() + }; + let size = bytes.len() as u64; + let found = candidates + .iter() + .copied() + .filter(|i| !present[*i].taken) + .min_by_key(|i| present[*i].size != size); + match found { + Some(i) => { + present[i].taken = true; + let wanted: BTreeSet = flags.iter().map(|f| f.to_lowercase()).collect(); + let have: BTreeSet = present[i] + .flags + .iter() + .filter(|f| *f != "\\recent") + .cloned() + .collect(); + if wanted == have { + counts.skipped += 1; + unchanged += 1; + continue; + } + if !dry_run { + client + .run_collect(&command::uid_store_flags(present[i].uid, &flags)) + .map_err(|e| folder_error(&folder.name, "STORE", e))?; + } + counts.updated += 1; + updated += 1; + } + None => { + if !dry_run { + let date = internal_date(delivery_secs(&email.received_at, &bytes)); + match client.append(&wire, &flags, Some(&date), &bytes) { + Ok(_) => {} + Err(ImapError::No(no)) => { + logger.warn(&format!( + "email {} into {:?}: refused: {}", + email.id, folder.name, no.text + )); + counts.failed += 1; + continue; + } + Err(e) => return Err(folder_error(&folder.name, "APPEND", e)), + } + } + counts.created += 1; + created += 1; + } + } + } + if dropped > 0 { + logger.warn(&format!( + "folder {:?}: the server keeps no keywords of its own here; \ + {dropped} were left off", + folder.name + )); + } + log_at( + logger, + &format!( + "{}: {created} appended, {updated} flags updated, {unchanged} already there", + folder.name + ), + ); + Ok(()) +} + +/// Selects the folder (read-only in a dry run) and reads what it holds: +/// each message's UID, flags, size and match headers. +fn read_folder( + client: &mut ImapClient, + wire: &str, + dry_run: bool, +) -> Result<(Vec, Allowed), ImapError> { + let selected = if dry_run { + client.run_collect(&command::examine(wire))? + } else { + client.run_collect(&command::select(wire))? + }; + let mut exists = 0; + let mut allowed = Allowed { + any_keyword: true, + listed: BTreeSet::new(), + }; + for u in &selected.untagged { + match u { + Untagged::Exists(n) => exists = *n, + Untagged::StatusLine(line) if line.code.as_deref() == Some("PERMANENTFLAGS") => { + let listed: BTreeSet = line + .code_args + .as_deref() + .unwrap_or("") + .trim_matches(['(', ')']) + .split_whitespace() + .map(str::to_lowercase) + .collect(); + allowed = Allowed { + any_keyword: listed.contains("\\*"), + listed, + }; + } + _ => {} + } + } + if exists == 0 { + return Ok((Vec::new(), allowed)); + } + let fetched = client.run_collect(&command::uid_fetch( + "1:*", + &["UID", "FLAGS", "RFC822.SIZE", MATCH_HEADERS], + ))?; + let present = fetched + .untagged + .iter() + .filter_map(|u| match u { + Untagged::Fetch { items, .. } => on_target(items), + _ => None, + }) + .collect(); + Ok((present, allowed)) +} + +fn on_target(items: &[(String, Value)]) -> Option { + let mut uid = None; + let mut flags = BTreeSet::new(); + let mut size = 0; + let mut headers: Vec = Vec::new(); + for (name, value) in items { + match (name.as_str(), value) { + ("UID", Value::Number(n)) => uid = u32::try_from(*n).ok(), + ("FLAGS", Value::List(list)) => { + flags = list + .iter() + .filter_map(|v| v.as_str().map(str::to_lowercase)) + .collect(); + } + ("RFC822.SIZE", Value::Number(n)) => size = *n, + (n, Value::Bytes(b)) if n.starts_with("BODY[HEADER") => headers = b.clone(), + (n, Value::Str(s)) if n.starts_with("BODY[HEADER") => headers = s.clone().into_bytes(), + _ => {} + } + } + if !headers.ends_with(b"\r\n\r\n") { + headers.extend_from_slice(b"\r\n"); + } + Some(OnTarget { + uid: uid?, + flags, + size, + index: email_index_from_blob(&headers), + taken: false, + }) +} + +/// The archive's keywords as IMAP flags, and how many the folder can't keep. +/// `$deleted` is left off, so nothing written is expunged by the next +/// client that expunges. +fn imap_flags(keywords: &[String], allowed: &Allowed) -> (Vec, u64) { + let mut out: Vec = Vec::new(); + let mut dropped = 0; + for keyword in keywords { + let flag = match keyword.to_ascii_lowercase().as_str() { + "$seen" => "\\Seen".to_owned(), + "$answered" => "\\Answered".to_owned(), + "$flagged" => "\\Flagged".to_owned(), + "$draft" => "\\Draft".to_owned(), + "$deleted" | "$recent" => continue, + _ if !is_atom(keyword) => { + dropped += 1; + continue; + } + lower if allowed.any_keyword || allowed.listed.contains(lower) => keyword.clone(), + _ => { + dropped += 1; + continue; + } + }; + if !out.iter().any(|f| f.eq_ignore_ascii_case(&flag)) { + out.push(flag); + } + } + (out, dropped) +} + +/// A flag keyword must be an IMAP atom: printable ASCII without the +/// characters IMAP gives a meaning. +fn is_atom(s: &str) -> bool { + !s.is_empty() + && !s.starts_with('\\') + && s.bytes().all(|b| { + (0x21..0x7f).contains(&b) + && !matches!(b, b'(' | b')' | b'{' | b'%' | b'*' | b'"' | b'\\' | b']') + }) +} + +fn internal_date(secs: i64) -> String { + let format = format_description!( + "[day padding:space]-[month repr:short]-[year] [hour]:[minute]:[second] +0000" + ); + OffsetDateTime::from_unix_timestamp(secs) + .unwrap_or(OffsetDateTime::UNIX_EPOCH) + .format(&format) + .unwrap_or_else(|_| " 1-Jan-1970 00:00:00 +0000".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn mailbox(id: i64, name: &str, parent_id: Option, role: Option<&str>) -> Mailbox { + Mailbox { + id, + name: name.to_owned(), + parent_id, + role: role.map(str::to_owned), + is_subscribed: true, + } + } + + fn target(name: &str, attributes: &[&str]) -> TargetFolder { + TargetFolder { + name: name.to_owned(), + attributes: attributes.iter().map(|s| s.to_string()).collect(), + } + } + + #[test] + fn folders_land_where_the_target_keeps_them() { + let plan = plan_folders( + &[ + mailbox(1, "Inbox", None, Some("inbox")), + mailbox(2, "Clients", Some(1), None), + mailbox(3, "Sent Items", None, Some("sent")), + mailbox(4, "Archive", None, Some("archive")), + mailbox(5, "2024", Some(4), None), + mailbox(6, "a/b", None, None), + ], + &[ + target("INBOX", &[]), + target("[Gmail]/Sent Mail", &["\\HasNoChildren", "\\Sent"]), + ], + "", + '/', + ); + let name = |id| plan[&id].name.as_str(); + assert_eq!(name(1), "INBOX"); + assert!(plan[&1].exists); + assert_eq!(name(2), "INBOX/Clients"); + assert_eq!(name(3), "[Gmail]/Sent Mail"); + assert!(plan[&3].exists); + assert_eq!(name(4), "Archive"); + assert!(!plan[&4].exists); + assert_eq!(plan[&4].special_use, Some("\\Archive")); + assert_eq!(name(5), "Archive/2024"); + assert_eq!(name(6), "a_b"); + } + + #[test] + fn a_namespace_prefix_holds_everything_but_the_inbox() { + let plan = plan_folders( + &[ + mailbox(1, "Inbox", None, Some("inbox")), + mailbox(2, "Work", None, None), + mailbox(3, "Q1", Some(2), None), + ], + &[target("INBOX", &[])], + "INBOX.", + '.', + ); + assert_eq!(plan[&1].name, "INBOX"); + assert_eq!(plan[&2].name, "INBOX.Work"); + assert_eq!(plan[&3].name, "INBOX.Work.Q1"); + } + + #[test] + fn keywords_become_flags_the_folder_can_keep() { + let kw = |l: &[&str]| l.iter().map(|s| s.to_string()).collect::>(); + let any = Allowed { + any_keyword: true, + listed: BTreeSet::new(), + }; + let (flags, dropped) = imap_flags( + &kw(&[ + "$seen", + "$Flagged", + "$deleted", + "$Junk", + "bad word", + "$answered", + ]), + &any, + ); + assert_eq!(flags, ["\\Seen", "\\Flagged", "$Junk", "\\Answered"]); + assert_eq!(dropped, 1); + let system_only = Allowed { + any_keyword: false, + listed: ["\\seen", "$forwarded"] + .iter() + .map(|s| s.to_string()) + .collect(), + }; + let (flags, dropped) = imap_flags(&kw(&["$seen", "$junk", "$Forwarded"]), &system_only); + assert_eq!(flags, ["\\Seen", "$Forwarded"]); + assert_eq!(dropped, 1); + } + + #[test] + fn internal_dates_are_imap_shaped() { + assert_eq!(internal_date(1_709_294_400), " 1-Mar-2024 12:00:00 +0000"); + assert_eq!(internal_date(1_734_567_890), "19-Dec-2024 00:24:50 +0000"); + } + + #[test] + fn a_target_message_is_read_from_its_fetch() { + let items = vec![ + ("UID".to_owned(), Value::Number(12)), + ( + "FLAGS".to_owned(), + Value::List(vec![ + Value::Atom("\\Seen".into()), + Value::Atom("$Junk".into()), + ]), + ), + ("RFC822.SIZE".to_owned(), Value::Number(321)), + ( + "BODY[HEADER.FIELDS (MESSAGE-ID FROM TO SUBJECT DATE)]".to_owned(), + Value::Bytes(b"Message-ID: \r\nSubject: hi\r\n\r\n".to_vec()), + ), + ]; + let t = on_target(&items).unwrap(); + assert_eq!(t.uid, 12); + assert_eq!(t.size, 321); + assert!(t.flags.contains("\\seen") && t.flags.contains("$junk")); + assert_eq!(t.index.mids, ["x@y"]); + } +} diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs index 4754d6a..aafb3de 100644 --- a/src/sync/import_imap/coordinator.rs +++ b/src/sync/import_imap/coordinator.rs @@ -484,7 +484,7 @@ pub struct Endpoint { pub implicit_tls: bool, } -fn parse_endpoint(url: &str) -> Result { +pub(crate) fn parse_endpoint(url: &str) -> Result { let parsed = Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; let scheme = parsed.scheme(); @@ -515,7 +515,7 @@ pub(super) fn authenticate_client(client: &mut ImapClient, auth: &ImapAuth) -> R authenticate(client, auth).map(|_| ()) } -fn authenticate(client: &mut ImapClient, auth: &ImapAuth) -> Result { +pub(crate) fn authenticate(client: &mut ImapClient, auth: &ImapAuth) -> Result { match auth { ImapAuth::Basic { user, password } => { if client.has_capability("AUTH=PLAIN") { @@ -568,7 +568,7 @@ fn is_auth_failed(e: &ImapError) -> bool { } } -fn discover_namespace(client: &mut ImapClient, logger: Logger) -> Result { +pub(crate) fn discover_namespace(client: &mut ImapClient, logger: Logger) -> Result { if !client.has_capability("NAMESPACE") { return Ok(String::new()); } diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 3974694..284e969 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -8,6 +8,7 @@ pub mod batch; pub mod emailmeta; pub mod export; +pub mod export_imap; pub mod import_dav; pub mod import_exchange_ews; pub mod import_exchange_graph; diff --git a/tests/integration_export_imap.rs b/tests/integration_export_imap.rs new file mode 100644 index 0000000..a5470e2 --- /dev/null +++ b/tests/integration_export_imap.rs @@ -0,0 +1,223 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-imap` into a real Dovecot, read back with `import imap`: folders +//! (nested, non-ASCII, one with the separator in its name), a message kept +//! in two folders, one with no Message-ID, standard and custom flags. A +//! second run writes nothing, and a changed flag is all a third run sends. + +mod integration; + +use std::collections::{BTreeMap, BTreeSet}; + +use integration::dovecot::Dovecot; +use integration::validate::{cleanup, common, open_archive, tmp_archive}; + +use inbuxa_migrate::db::{blobs, init}; +use inbuxa_migrate::sync::export_imap::{self, ExportImapConfig}; +use inbuxa_migrate::sync::import_imap::{self, ImapAuth, ImapImportConfig}; +use inbuxa_migrate::sync::{Summary, TypeCounts}; +use rusqlite::{Connection, params}; + +fn message(subject: &str, with_id: bool) -> Vec { + let id = if with_id { + format!( + "Message-ID: <{}@export-imap.test>\r\n", + subject.replace(' ', "-") + ) + } else { + String::new() + }; + format!( + "From: dave@example.net\r\nTo: user@example.org\r\nSubject: {subject}\r\n{id}\ + Date: Fri, 01 Mar 2024 12:00:00 +0000\r\n\r\n{subject}\r\n" + ) + .into_bytes() +} + +fn seed(path: &std::path::Path) { + let conn = init::open(path).unwrap(); + for (id, name, parent, role) in [ + (1, "Inbox", None, Some("inbox")), + (2, "Clients", Some(1), None), + (3, "Archive", None, Some("archive")), + (4, "2024", Some(3), None), + (5, "Équipe & co", None, None), + (6, "Q1/Q2", None, None), + ] { + conn.execute( + "INSERT INTO mailboxes (id, name, parent_id, role) VALUES (?1, ?2, ?3, ?4)", + params![id, name, parent, role], + ) + .unwrap(); + } + for (subject, with_id, mailboxes, keywords) in [ + ("Seen and flagged", true, "[1]", r#"["$seen","$flagged"]"#), + ("Junk in two folders", true, "[1,3]", r#"["$Junk"]"#), + ("Client note", true, "[2]", r#"["$answered","project-x"]"#), + ("Old report", true, "[4]", "[]"), + ("Bonjour", true, "[5]", r#"["$seen"]"#), + ("No message id", false, "[6]", r#"["$seen"]"#), + ] { + let blob = blobs::intern_blob(&conn, &message(subject, with_id)).unwrap(); + conn.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-03-01T12:00:00Z', ?2, ?3)", + params![blob, mailboxes, keywords], + ) + .unwrap(); + } +} + +fn counts<'a>(summary: &'a Summary, ty: &str) -> &'a TypeCounts { + &summary.per_type.iter().find(|(t, _)| *t == ty).unwrap().1 +} + +/// Each message as (folder path, subject) → keywords, lowercased. +fn mail_by_folder(conn: &Connection) -> BTreeMap<(String, String), BTreeSet> { + let rows: Vec<(i64, String, Option, Option)> = conn + .prepare("SELECT id, name, parent_id, role FROM mailboxes") + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + let name_of = |id: i64| { + let (_, name, _, role) = rows.iter().find(|r| r.0 == id).unwrap(); + if role.as_deref() == Some("inbox") { + "INBOX".to_owned() + } else { + name.clone() + } + }; + let path_of = |id: i64| { + let mut parts = vec![name_of(id)]; + let mut parent = rows.iter().find(|r| r.0 == id).unwrap().2; + while let Some(p) = parent { + parts.insert(0, name_of(p)); + parent = rows.iter().find(|r| r.0 == p).unwrap().2; + } + parts.join("/") + }; + let mut out = BTreeMap::new(); + let mut stmt = conn + .prepare("SELECT b.data, e.mailbox_ids, e.keywords FROM emails e JOIN blobs b ON b.id = e.blob_id") + .unwrap(); + let rows_e = stmt + .query_map([], |r| { + Ok(( + r.get::<_, Vec>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + }) + .unwrap(); + for row in rows_e { + let (data, mailboxes, keywords) = row.unwrap(); + let text = String::from_utf8(data).unwrap(); + let subject = text + .lines() + .find_map(|l| l.strip_prefix("Subject: ")) + .unwrap() + .to_owned(); + let keywords: BTreeSet = serde_json::from_str::>(&keywords) + .unwrap() + .into_iter() + .map(|k| k.to_lowercase()) + .collect(); + for mailbox in serde_json::from_str::>(&mailboxes).unwrap() { + out.insert((path_of(mailbox), subject.clone()), keywords.clone()); + } + } + out +} + +#[test] +#[ignore = "requires Docker"] +fn dovecot_export_imap_round_trip_converges() { + let d = Dovecot::start().expect("dovecot start"); + let account = d.accounts.last().expect("account").clone(); + let url = format!("imap://{}:{}", d.imap.host, d.imap.port); + let auth = ImapAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }; + let config = || ExportImapConfig { + url: url.clone(), + auth: auth.clone(), + allow_cleartext: false, + }; + + let archive = tmp_archive("export-imap-src"); + seed(&archive); + + let first = export_imap::run(common(&archive), config()).expect("first export"); + assert!(!first.any_failed(), "{first:?}"); + assert_eq!(counts(&first, "email").created, 7, "{first:?}"); + assert_eq!(counts(&first, "mailbox").created, 5, "{first:?}"); + + let again = export_imap::run(common(&archive), config()).expect("second export"); + let email = counts(&again, "email"); + assert_eq!( + (email.created, email.updated, email.skipped), + (0, 0, 7), + "{again:?}" + ); + assert_eq!(counts(&again, "mailbox").created, 0); + + let back = tmp_archive("export-imap-back"); + let imported = import_imap::run( + common(&back), + ImapImportConfig { + url: url.clone(), + auth: auth.clone(), + allow_cleartext: false, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 64, + fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, + imap_connections: 1, + allow_source_change: false, + }, + ) + .expect("import back"); + assert!(!imported.any_failed(), "{imported:?}"); + + let expected: BTreeMap<_, _> = mail_by_folder(&open_archive(&archive)) + .into_iter() + // The target's separator is `/`, so it can't sit inside a name. + .map(|((path, subject), kw)| ((path.replace("Q1/Q2", "Q1_Q2"), subject), kw)) + .collect(); + assert_eq!(mail_by_folder(&open_archive(&back)), expected); + let received: String = open_archive(&back) + .query_row("SELECT received_at FROM emails LIMIT 1", [], |r| r.get(0)) + .unwrap(); + assert_eq!(received, "2024-03-01T12:00:00Z"); + + open_archive(&archive) + .execute( + "UPDATE emails SET keywords = '[\"$seen\"]' WHERE keywords LIKE '%flagged%'", + [], + ) + .unwrap(); + let third = export_imap::run(common(&archive), config()).expect("third export"); + let email = counts(&third, "email"); + assert_eq!( + (email.created, email.updated, email.skipped), + (0, 1, 6), + "{third:?}" + ); + + cleanup(&archive); + cleanup(&back); + d.stop().expect("dovecot stop"); +} diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs index a03f0e7..394c167 100644 --- a/tests/mock_imap.rs +++ b/tests/mock_imap.rs @@ -2221,3 +2221,90 @@ fn a_chunk_larger_than_the_event_queue_completes() { let summary = run_import(&server, "alice", archive, |_| {}).expect("import"); assert_eq!(email_counts(&summary).created, 10, "summary={summary:?}"); } + +fn append_greeting(conn: &mut MockConn, caps: &str) -> std::io::Result<()> { + conn.write_line("* OK ready")?; + let (tag, _) = conn.read_command()?; + conn.write_line(&format!("* CAPABILITY {caps}"))?; + conn.write_line(&format!("{tag} OK done")) +} + +fn literal_len(cmd: &str) -> usize { + let open = cmd.rfind('{').expect("literal"); + cmd[open + 1..cmd.len() - 1] + .trim_end_matches('+') + .parse() + .expect("length") +} + +#[test] +fn append_waits_for_the_go_ahead_without_literal_plus() { + let message: &'static [u8] = b"Subject: hi\r\n\r\nbody\r\n"; + let server = MockImap::start(move |conn| { + use std::io::Read; + append_greeting(conn, "IMAP4rev1 AUTH=PLAIN")?; + let (tag, cmd) = conn.read_command()?; + assert!( + cmd.starts_with("APPEND \"INBOX\" (\\Seen) \" 1-Mar-2024 12:00:00 +0000\" {"), + "{cmd}" + ); + assert!(cmd.ends_with('}') && !cmd.ends_with("+}"), "{cmd}"); + let mut body = vec![0u8; literal_len(&cmd) + 2]; + conn.write_line("+ go ahead")?; + conn.reader.read_exact(&mut body)?; + assert_eq!(&body[..body.len() - 2], message); + assert_eq!(&body[body.len() - 2..], b"\r\n"); + conn.write_line(&format!("{tag} OK [APPENDUID 1 5] done"))?; + drain_until_close(conn); + Ok(()) + }); + let mut client = connect_mock(&server).expect("connect"); + let line = client + .append( + "INBOX", + &["\\Seen".to_owned()], + Some(" 1-Mar-2024 12:00:00 +0000"), + message, + ) + .expect("append"); + assert_eq!(line.code.as_deref(), Some("APPENDUID")); +} + +#[test] +fn append_sends_at_once_with_literal_plus() { + let message: &'static [u8] = b"Subject: hi\r\n\r\nbody\r\n"; + let server = MockImap::start(move |conn| { + use std::io::Read; + append_greeting(conn, "IMAP4rev1 LITERAL+")?; + let (tag, cmd) = conn.read_command()?; + assert!(cmd.ends_with("+}"), "{cmd}"); + let mut body = vec![0u8; literal_len(&cmd) + 2]; + conn.reader.read_exact(&mut body)?; + assert_eq!(&body[..body.len() - 2], message); + conn.write_line(&format!("{tag} OK done"))?; + drain_until_close(conn); + Ok(()) + }); + let mut client = connect_mock(&server).expect("connect"); + client.append("INBOX", &[], None, message).expect("append"); +} + +#[test] +fn append_refused_before_the_go_ahead_sends_nothing_and_keeps_the_session() { + let server = MockImap::start(|conn| { + append_greeting(conn, "IMAP4rev1")?; + let (tag, _) = conn.read_command()?; + conn.write_line(&format!("{tag} NO [TOOBIG] message too large"))?; + let (tag, cmd) = conn.read_command()?; + assert_eq!(cmd, "NOOP", "the literal must not follow a NO"); + conn.write_line(&format!("{tag} OK noop"))?; + drain_until_close(conn); + Ok(()) + }); + let mut client = connect_mock(&server).expect("connect"); + match client.append("INBOX", &[], None, b"x\r\n") { + Err(ImapError::No(no)) => assert!(no.text.contains("too large"), "{no:?}"), + other => panic!("expected NO, got {other:?}"), + } + client.noop().expect("session still usable"); +} -- 2.54.0