From fe96d7d2342084c166f307d7a60f640dabe64de7 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 29 Oct 2026 22:37:08 -0700 Subject: [PATCH] Add export-files: an archive as Maildir, vCard, iCalendar and Sieve files The way out of an archive for anything that doesn't speak JMAP: mail as a Maildir++ tree Dovecot and Courier serve as it is, each address book as a .vcf, each calendar as one .ics, Sieve scripts, the file tree, and a manifest.json with what those formats don't hold (real folder names, roles, the active script, identities, links). No network; a new or empty directory only, private to its owner; every name from the source made safe as a path. Photos and attachments are inlined as data: URIs, as export sends them. Links are listed, never made. The readers it round-trips through get what that needs: - import maildir decodes modified UTF-7 folder names and reads each folder's dovecot-keywords, so Junk and labels come back; - import takeout unfolds and unescapes X-WR-CALNAME and X-WR-CALDESC. tests/export_files.rs writes an archive out and reads it back with import maildir and import takeout. --- CHANGELOG.md | 16 + README.md | 8 +- docs/usage.md | 58 +++- src/cli.rs | 44 +++ src/export_files/cards.rs | 479 +++++++++++++++++++++++++++ src/export_files/mail.rs | 485 ++++++++++++++++++++++++++++ src/export_files/mod.rs | 419 ++++++++++++++++++++++++ src/export_files/paths.rs | 111 +++++++ src/lib.rs | 1 + src/main.rs | 6 +- src/sync/import_maildir/keywords.rs | 76 +++++ src/sync/import_maildir/messages.rs | 23 +- src/sync/import_maildir/tree.rs | 17 +- src/sync/import_takeout/calendar.rs | 50 ++- tests/export_files.rs | 478 +++++++++++++++++++++++++++ 15 files changed, 2256 insertions(+), 15 deletions(-) create mode 100644 src/export_files/cards.rs create mode 100644 src/export_files/mail.rs create mode 100644 src/export_files/mod.rs create mode 100644 src/export_files/paths.rs create mode 100644 tests/export_files.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ba5080e..b6b47d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ inbuxa suite: a series (`27.1`), its patches (`27.1.3`) and release candidates before a series ships (`27.1-rc.1`). The tag is the version with a `v`. Releases before 27.1 were dates (`v2026.9.30`). +## [Unreleased] + +### Added +- `export-files` writes an archive out as plain files in open formats: mail + as a Maildir++ tree, each address book as a `.vcf`, each calendar as an + `.ics`, Sieve scripts, the file tree, and a `manifest.json` with what those + formats don't hold. It needs no network, writes only into a new or empty + directory, and keeps the directory private to its owner. + +### Changed +- `import maildir` reads folder names in modified UTF-7, as Dovecot and + Courier write them, and the custom keywords in each folder's + `dovecot-keywords` file (Junk, labels). +- `import takeout` unfolds and unescapes a calendar's `X-WR-CALNAME` and + `X-WR-CALDESC`, so a long name or one with a comma reads back whole. + ## [27.1-rc.1] The first release candidate of the 27.1 series, with the rest of the suite. diff --git a/README.md b/README.md index 6178b97..a5e5ed7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate import imap … alice.sqlite an account into an archive inbuxa-migrate inspect alice.sqlite what landed inbuxa-migrate export … alice.sqlite the archive into inbuxa + inbuxa-migrate export-files alice.sqlite out the archive as plain files Between the two sides sits an archive: one SQLite file holding one account. `import` fills it from the old server; `export` writes it into the new one. @@ -33,8 +34,11 @@ restore it later with `export`. An archive remembers which account filled it and refuses a different one unless told otherwise. Export speaks JMAP only. It writes into inbuxa, or into any JMAP server that -advertises the types being written. The full command reference is in -[docs/usage.md](docs/usage.md). +advertises the types being written. `export-files` is the way out to +everything else: it writes the archive as a Maildir++ tree, vCard, +iCalendar and Sieve files that other servers and mail, calendar and +contacts apps open, with no network and no store format in the way. The full +command reference is in [docs/usage.md](docs/usage.md). ## Sources diff --git a/docs/usage.md b/docs/usage.md index 0c7bbc5..d7820aa 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,7 +13,7 @@ and how to install it. ## Commands and global flags ``` -inbuxa-migrate [args...] +inbuxa-migrate [args...] ``` Every command takes a few global flags -- verbosity, worker pool size, retry @@ -259,7 +259,8 @@ type ends with a line of what was created, updated, left unchanged and failed. `--prune` also deletes what is on the target and not in the archive. It asks -first; `--yes` answers for it, for scripts. Export speaks JMAP only. +first; `--yes` answers for it, for scripts. Export speaks JMAP only; for +anything else, see [Export to files](#export-to-files). With `--dry-run`, export reads the target and writes nothing to the account. It prints the plan in plain words: for each type, how much would be @@ -280,6 +281,59 @@ The plan lists each predicted failure and its reason. When anything would fail, the dry run exits 5, as the real run would, so a script can stop before it starts. +## Export to files + +``` +inbuxa-migrate export-files [--objects ] [--dry-run] +``` + +Writes `ARCHIVE` out as plain files in open formats, for a server or app +that doesn't speak JMAP, or to keep a copy that needs no tool to read. It +never opens a network connection. `DIR` must be new or empty, and is made +readable by its owner only, like the archive. + +``` +DIR/ + mail/ Maildir++: the inbox here, other folders as .Parent.Child + contacts/*.vcf one file per address book + calendars/*.ics one file per calendar + sieve/*.sieve one file per script + files/ the file tree + manifest.json what the formats above don't hold +``` + +- **Mail** is laid out as Dovecot and Courier keep it, so either can serve + it as it is, and `import maildir` reads it back. Folder names are in + modified UTF-7. A `.` or `/` inside a folder's name becomes `_` on disk, + since Maildir++ uses the dot between levels. A message kept in several + folders is written to each. Read, answered, flagged, draft, forwarded and + deleted are the standard Maildir flags; other keywords (Junk, labels) are + the letters `a` to `z`, named in each folder's `dovecot-keywords`, up to + 26 per folder. Each file is dated when the message arrived. +- **Contacts** are vCard 4.0, and a card in two address books is in both + files. **Calendars** are iCalendar, one VCALENDAR each, named by + `X-WR-CALNAME`, with each time zone once. Photos and attachments the + archive holds go inside as `data:` URIs, as `export` sends them. + `import takeout` reads both back. +- **Sieve scripts** are written as they are; the manifest says which is + active. +- **Files** keep their names and dates. Links are listed in the manifest + and never made, since the source chose where they point. + +Every name comes from the source, so none is used as a path as it stands: +separators and names Windows refuses become `_`, and names that clash, even +only by case, are numbered. + +`manifest.json` records the source account, every folder's real name, +path, role and message count, each address book and calendar with its +file, color and time zone, which Sieve script is active, the identities, +and the links. It is written last: a tree without one was interrupted, and +the way to finish it is to run again into an empty directory. + +`--objects` takes the same types as `export`; naming a folder or a message +type writes all of mail, and the same for contacts and calendars. With +`--dry-run` it counts what it would write and creates nothing. + ## Inspect ``` diff --git a/src/cli.rs b/src/cli.rs index d21278b..2a832ef 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -14,6 +14,7 @@ use crate::error::Error; use crate::exchange_ews::oauth::OAuthFlow; use crate::exchange_ews::types::MailboxKind; use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces}; +use crate::export_files::ExportFilesConfig; use crate::inspect::InspectConfig; use crate::jmap::account::AccountSelector; use crate::jmap::http::Auth; @@ -49,6 +50,11 @@ enum Command { }, #[command(about = "Push a local archive into a target JMAP server")] Export(Box), + #[command( + about = "Write a local archive out as plain files: Maildir, vCard, iCalendar, Sieve", + name = "export-files" + )] + ExportFiles(Box), #[command(about = "Show the contents of a local archive (read-only)")] Inspect(Box), } @@ -359,6 +365,31 @@ struct ExportArgs { archive: PathBuf, } +#[derive(Args)] +struct ExportFilesArgs { + #[arg( + long, + value_name = "LIST", + help = "Types to write (default: everything in the archive)" + )] + objects: Option, + + #[arg(long, help = "Count what would be written; write nothing")] + dry_run: bool, + + #[arg(short = 'v', long, action = ArgAction::Count, conflicts_with = "quiet", help = "Increase log verbosity")] + verbose: u8, + + #[arg(short = 'q', long, help = "Quiet: warnings and errors only")] + quiet: bool, + + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, + + #[arg(value_name = "DIR", help = "Directory to write, new or empty")] + dir: PathBuf, +} + #[derive(Args)] struct InspectArgs { #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] @@ -396,6 +427,7 @@ pub enum Action { ImportExchangeEws(CommonConfig, EwsImportConfig), ImportExchangeGraph(CommonConfig, GraphImportConfig), Export(CommonConfig, ExportConfig), + ExportFiles(ExportFilesConfig), Inspect(InspectConfig), } @@ -521,6 +553,7 @@ impl Cli { match self.command { Command::Import { source } => resolve_import(source), Command::Export(args) => resolve_export(*args), + Command::ExportFiles(args) => resolve_export_files(*args), Command::Inspect(args) => resolve_inspect(*args), } } @@ -899,6 +932,17 @@ fn resolve_export(args: ExportArgs) -> Result { )) } +fn resolve_export_files(args: ExportFilesArgs) -> Result { + let objects = args.objects.as_deref().map(parse_object_list).transpose()?; + Ok(Action::ExportFiles(ExportFilesConfig { + archive: args.archive, + dir: args.dir, + objects, + dry_run: args.dry_run, + logger: Logger::from_flags(args.quiet, args.verbose), + })) +} + fn resolve_inspect(args: InspectArgs) -> Result { let target = match args.target.as_deref() { Some(token) => Some(ObjectType::parse(token)?), diff --git a/src/export_files/cards.rs b/src/export_files/cards.rs new file mode 100644 index 0000000..a0f52d0 --- /dev/null +++ b/src/export_files/cards.rs @@ -0,0 +1,479 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Address books as `.vcf` files and calendars as `.ics` files, one per +//! collection, converted back from the JSContact and JSCalendar the archive +//! holds. + +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::Path; + +use calcard::jscalendar::JSCalendar; +use calcard::jscontact::JSContact; +use rusqlite::Connection; +use serde_json::{Value, json}; + +use crate::db::blobs; +use crate::error::Error; +use crate::jmap::blob::{BlobWalkError, InlineShape, inline_blob_data_uris}; +use crate::jmap::error::JmapError; +use crate::logging::Logger; +use crate::sync::TypeCounts; + +use super::paths::{Names, safe_component}; + +pub struct Outcome { + pub collection: TypeCounts, + pub item: TypeCounts, + pub manifest: Value, +} + +struct Collection { + id: i64, + name: String, + description: Option, + color: Option, + time_zone: Option, + is_default: bool, +} + +pub fn write_contacts( + conn: &Connection, + root: &Path, + dry_run: bool, + logger: Logger, +) -> Result { + let books = read_collections( + conn, + "SELECT id, name, description, NULL, NULL, is_default FROM address_books ORDER BY sort_order, id", + )?; + let mut per_book: HashMap> = HashMap::new(); + let mut item = TypeCounts::default(); + let mut stmt = + conn.prepare("SELECT id, uid, address_book_ids, data FROM contact_cards ORDER BY id")?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })?; + for row in rows { + let (id, uid, book_ids, data) = row?; + let text = match vcard_text(conn, &uid, &data) { + Ok(text) => text, + Err(reason) => { + logger.warn(&format!( + "contact {id} ({uid}): not written as a vCard: {reason}" + )); + item.failed += 1; + continue; + } + }; + let book_ids: Vec = serde_json::from_str(&book_ids).unwrap_or_default(); + for book in book_ids + .iter() + .filter(|b| books.iter().any(|c| c.id == **b)) + { + per_book.entry(*book).or_default().push(text.clone()); + } + item.created += 1; + } + + let mut names = Names::default(); + let mut manifest = Vec::new(); + for book in &books { + let file = names.unique(&safe_component(&book.name), ".vcf"); + let cards = per_book.remove(&book.id).unwrap_or_default(); + if !dry_run { + fs::create_dir_all(root)?; + fs::write(root.join(&file), cards.concat())?; + } + manifest.push(json!({ + "name": book.name, + "description": book.description, + "file": format!("contacts/{file}"), + "default": book.is_default, + "cards": cards.len(), + })); + } + Ok(Outcome { + collection: TypeCounts { + created: books.len() as u64, + ..Default::default() + }, + item, + manifest: Value::Array(manifest), + }) +} + +pub fn write_calendars( + conn: &Connection, + root: &Path, + dry_run: bool, + logger: Logger, +) -> Result { + let calendars = read_collections( + conn, + "SELECT id, name, description, color, time_zone, is_default FROM calendars ORDER BY sort_order, id", + )?; + let mut per_calendar: HashMap> = HashMap::new(); + let mut item = TypeCounts::default(); + let mut stmt = + conn.prepare("SELECT id, calendar_ids, data, data_type FROM calendar_events ORDER BY id")?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })?; + for row in rows { + let (id, calendar_ids, data, data_type) = row?; + let text = match icalendar_text(conn, &data, &data_type) { + Ok(text) => text, + Err(reason) => { + logger.warn(&format!("event {id}: not written as iCalendar: {reason}")); + item.failed += 1; + continue; + } + }; + let calendar_ids: Vec = serde_json::from_str(&calendar_ids).unwrap_or_default(); + for calendar in calendar_ids + .iter() + .filter(|c| calendars.iter().any(|k| k.id == **c)) + { + per_calendar + .entry(*calendar) + .or_default() + .push(text.clone()); + } + item.created += 1; + } + + let mut names = Names::default(); + let mut manifest = Vec::new(); + for calendar in &calendars { + let file = names.unique(&safe_component(&calendar.name), ".ics"); + let events = per_calendar.remove(&calendar.id).unwrap_or_default(); + if !dry_run { + fs::create_dir_all(root)?; + fs::write(root.join(&file), merge_calendar(calendar, &events))?; + } + manifest.push(json!({ + "name": calendar.name, + "description": calendar.description, + "color": calendar.color, + "timeZone": calendar.time_zone, + "file": format!("calendars/{file}"), + "default": calendar.is_default, + "events": events.len(), + })); + } + Ok(Outcome { + collection: TypeCounts { + created: calendars.len() as u64, + ..Default::default() + }, + item, + manifest: Value::Array(manifest), + }) +} + +fn read_collections(conn: &Connection, sql: &str) -> Result, Error> { + let mut stmt = conn.prepare(sql)?; + let rows = stmt.query_map([], |row| { + Ok(Collection { + id: row.get(0)?, + name: row.get(1)?, + description: row.get(2)?, + color: row.get(3)?, + time_zone: row.get(4)?, + is_default: row.get::<_, i64>(5)? != 0, + }) + })?; + Ok(rows.collect::>()?) +} + +/// The archive keeps a card's uid beside its JSContact, and photos and +/// other media as blobs; the vCard needs both back inside. +fn vcard_text(conn: &Connection, uid: &str, data: &str) -> Result { + let mut value: Value = serde_json::from_str(data).map_err(|e| e.to_string())?; + let map = value.as_object_mut().ok_or("not a JSON object")?; + map.insert("uid".to_owned(), Value::String(uid.to_owned())); + map.entry("@type").or_insert_with(|| json!("Card")); + inline_blobs(conn, &mut value, InlineShape::JsContactResource)?; + let json = value.to_string(); + let card = JSContact::::parse(&json)?; + Ok(card.into_vcard().ok_or("no vCard came of it")?.to_string()) +} + +/// calcard writes a VCALENDAR from a Group, so each event goes in one, with +/// its attachments inlined from the archive's blobs. +fn icalendar_text(conn: &Connection, data: &str, data_type: &str) -> Result { + let mut value: Value = serde_json::from_str(data).map_err(|e| e.to_string())?; + value + .as_object_mut() + .ok_or("not a JSON object")? + .entry("@type") + .or_insert_with(|| json!(data_type)); + inline_blobs(conn, &mut value, InlineShape::JsCalendarLink)?; + let group = if data_type == "Group" { + value + } else { + json!({ "@type": "Group", "entries": [value] }) + }; + let json = group.to_string(); + let event = JSCalendar::::parse(&json)?; + Ok(event + .into_icalendar() + .ok_or("no iCalendar came of it")? + .to_string()) +} + +/// Blobs the archive refers to, as `data:` URIs, the way `export` sends them. +fn inline_blobs(conn: &Connection, value: &mut Value, shape: InlineShape) -> Result<(), String> { + inline_blob_data_uris(value, shape, |local_id| { + match blobs::blob_bytes(conn, local_id) { + Ok(Some(bytes)) => Ok(bytes), + Ok(None) => Err(BlobWalkError::resolver(JmapError::Transport(format!( + "blob {local_id} is missing from the archive" + )))), + Err(e) => Err(BlobWalkError::resolver(JmapError::Transport(e.to_string()))), + } + }) + .map_err(|e| e.to_string()) +} + +/// One VCALENDAR for a whole calendar: its name and details as the +/// `X-WR-*` properties calendar apps read, then every event's components, +/// each time zone once. +fn merge_calendar(calendar: &Collection, events: &[String]) -> String { + let mut out = String::new(); + out.push_str("BEGIN:VCALENDAR\r\nVERSION:2.0\r\n"); + out.push_str("PRODID:-//Coffey Labs//inbuxa-migrate//EN\r\n"); + push_property(&mut out, "X-WR-CALNAME", &calendar.name); + if let Some(description) = calendar.description.as_deref().filter(|d| !d.is_empty()) { + push_property(&mut out, "X-WR-CALDESC", description); + } + if let Some(tz) = calendar.time_zone.as_deref().filter(|t| !t.is_empty()) { + push_property(&mut out, "X-WR-TIMEZONE", tz); + } + if let Some(color) = calendar.color.as_deref().filter(|c| !c.is_empty()) { + push_property(&mut out, "X-APPLE-CALENDAR-COLOR", color); + } + let mut zones: HashSet = HashSet::new(); + for event in events { + for component in components(event) { + if let Some(tzid) = component.tzid + && !zones.insert(tzid) + { + continue; + } + out.push_str(&component.text); + } + } + out.push_str("END:VCALENDAR\r\n"); + out +} + +struct Component { + tzid: Option, + text: String, +} + +/// The components directly inside an iCalendar object's VCALENDAR, each +/// with its lines as they were (folded lines stay folded). +fn components(ics: &str) -> Vec { + let mut out = Vec::new(); + let mut depth = 0usize; + let mut current: Option<(String, Component)> = None; + for line in ics.split_inclusive('\n') { + let bare = line.trim_end_matches(['\r', '\n']); + let upper = bare.to_ascii_uppercase(); + let continuation = bare.starts_with([' ', '\t']); + if !continuation && upper.starts_with("BEGIN:") { + depth += 1; + if depth == 2 { + current = Some(( + upper[6..].to_owned(), + Component { + tzid: None, + text: String::new(), + }, + )); + } + } + if depth >= 2 + && let Some((name, component)) = current.as_mut() + { + component.text.push_str(bare); + component.text.push_str("\r\n"); + if depth == 2 && name == "VTIMEZONE" && !continuation && upper.starts_with("TZID") { + component.tzid = bare.split_once(':').map(|(_, v)| v.to_owned()); + } + } + if !continuation && upper.starts_with("END:") { + if depth == 2 + && let Some((_, component)) = current.take() + { + out.push(component); + } + depth = depth.saturating_sub(1); + } + } + out +} + +fn push_property(out: &mut String, name: &str, value: &str) { + let mut escaped = String::with_capacity(value.len()); + for c in value.chars() { + match c { + '\\' => escaped.push_str("\\\\"), + ';' => escaped.push_str("\\;"), + ',' => escaped.push_str("\\,"), + '\n' => escaped.push_str("\\n"), + '\r' => {} + c => escaped.push(c), + } + } + out.push_str(&fold(&format!("{name}:{escaped}"))); +} + +/// RFC 5545 folding: lines of at most 75 octets, never splitting a +/// character. +fn fold(line: &str) -> String { + let mut out = String::with_capacity(line.len() + 8); + let mut width = 0; + for c in line.chars() { + let len = c.len_utf8(); + if width + len > 75 { + out.push_str("\r\n "); + width = 1; + } + out.push(c); + width += len; + } + out.push_str("\r\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const EVENT_A: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\n\ +BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nBEGIN:STANDARD\r\nDTSTART:19701025T030000\r\n\ +TZOFFSETFROM:+0200\r\nTZOFFSETTO:+0100\r\nEND:STANDARD\r\nEND:VTIMEZONE\r\n\ +BEGIN:VEVENT\r\nUID:a\r\nSUMMARY:A long\r\n folded summary\r\nBEGIN:VALARM\r\nACTION:DISPLAY\r\n\ +END:VALARM\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; + + const EVENT_B: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ +BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nEND:VTIMEZONE\r\n\ +BEGIN:VEVENT\r\nUID:b\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; + + fn archive() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init::apply_schema(&conn).unwrap(); + conn + } + + #[test] + fn a_photo_kept_as_a_blob_goes_into_the_vcard() { + let conn = archive(); + let blob = blobs::intern_blob(&conn, b"\x89PNG fake").unwrap(); + let data = json!({ + "@type": "Card", + "name": { "full": "Carol" }, + "media": { "p": { "kind": "photo", "@blob": blob, "mediaType": "image/png" } } + }); + let text = vcard_text(&conn, "card-carol", &data.to_string()).unwrap(); + assert!(text.contains("PHOTO"), "{text}"); + // calcard escapes the comma inside the URI, as it does for CardDAV. + assert!(text.contains("data:image/png;base64"), "{text}"); + assert!(!text.contains("@blob"), "{text}"); + } + + #[test] + fn a_missing_blob_fails_the_item_with_a_reason() { + let data = json!({ "@type": "Card", "media": { "p": { "kind": "photo", "@blob": 99 } } }); + let err = vcard_text(&archive(), "x", &data.to_string()).unwrap_err(); + assert!(err.contains("blob 99"), "{err}"); + } + + fn calendar() -> Collection { + Collection { + id: 1, + name: "Team, Berlin; ops".to_owned(), + description: Some("Line one\nline two".to_owned()), + color: Some("#ff0000".to_owned()), + time_zone: Some("Europe/Berlin".to_owned()), + is_default: true, + } + } + + #[test] + fn events_merge_into_one_calendar_with_each_zone_once() { + let ics = merge_calendar(&calendar(), &[EVENT_A.to_owned(), EVENT_B.to_owned()]); + assert!(ics.starts_with("BEGIN:VCALENDAR\r\nVERSION:2.0\r\n")); + assert!(ics.ends_with("END:VCALENDAR\r\n")); + assert_eq!(ics.matches("BEGIN:VCALENDAR").count(), 1); + assert_eq!(ics.matches("BEGIN:VTIMEZONE").count(), 1); + assert_eq!(ics.matches("BEGIN:VEVENT").count(), 2); + assert_eq!(ics.matches("PRODID:").count(), 1); + assert!(ics.contains("SUMMARY:A long\r\n folded summary\r\n")); + assert!(ics.contains("BEGIN:VALARM\r\nACTION:DISPLAY\r\nEND:VALARM\r\n")); + assert!(ics.contains("X-WR-CALNAME:Team\\, Berlin\\; ops\r\n")); + assert!(ics.contains("X-WR-CALDESC:Line one\\nline two\r\n")); + assert!(ics.contains("X-WR-TIMEZONE:Europe/Berlin\r\n")); + } + + #[test] + fn long_lines_fold_on_character_boundaries() { + let folded = fold(&format!("X-WR-CALNAME:{}", "é".repeat(60))); + for line in folded.split("\r\n").filter(|l| !l.is_empty()) { + assert!(line.len() <= 75, "{} octets", line.len()); + } + let unfolded = folded.replace("\r\n ", ""); + assert_eq!(unfolded, format!("X-WR-CALNAME:{}\r\n", "é".repeat(60))); + } + + #[test] + fn a_card_round_trips_through_jscontact() { + let card = crate::sync::import_dav::calcard::vcard_to_jscontact( + "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:abc-123\r\nFN:Alice Smith\r\n\ +EMAIL:alice@example.com\r\nEND:VCARD\r\n", + "/x", + ) + .unwrap(); + let text = vcard_text(&archive(), &card.uid, &card.data.to_string()).unwrap(); + assert!(text.contains("VERSION:4.0"), "{text}"); + assert!(text.contains("UID:abc-123"), "{text}"); + assert!(text.contains("FN:Alice Smith"), "{text}"); + assert!(text.contains("alice@example.com"), "{text}"); + } + + #[test] + fn an_event_round_trips_through_jscalendar() { + let entries = crate::sync::import_dav::calcard::ical_to_jscalendar_entries( + "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\nBEGIN:VEVENT\r\nUID:ev-1\r\n\ +DTSTAMP:20240101T000000Z\r\nDTSTART;TZID=Europe/Berlin:20240301T090000\r\n\ +DURATION:PT1H\r\nRRULE:FREQ=WEEKLY;COUNT=4\r\nSUMMARY:Standup\r\nEND:VEVENT\r\n\ +END:VCALENDAR\r\n", + ) + .unwrap(); + let mut data = entries[0].data.clone(); + crate::sync::import_dav::calcard::strip_extracted_fields_from_event(&mut data); + let text = icalendar_text(&archive(), &data.to_string(), "Event").unwrap(); + assert!(text.contains("UID:ev-1"), "{text}"); + assert!(text.contains("SUMMARY:Standup"), "{text}"); + assert!(text.contains("FREQ=WEEKLY"), "{text}"); + assert!(text.contains("Europe/Berlin"), "{text}"); + } +} diff --git a/src/export_files/mail.rs b/src/export_files/mail.rs new file mode 100644 index 0000000..46a4f9c --- /dev/null +++ b/src/export_files/mail.rs @@ -0,0 +1,485 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Mail as a Maildir++ tree, the layout Dovecot and Courier serve and +//! `import maildir` reads: the inbox at the root, every other folder a +//! `.Parent.Child` directory beside it, names in modified UTF-7, flags in +//! the file name and other keywords in each folder's `dovecot-keywords`. + +use std::collections::HashMap; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use encodify::utf7; +use rusqlite::Connection; +use serde_json::{Value, json}; +use time::OffsetDateTime; +use time::format_description::well_known::{Rfc2822, Rfc3339}; + +use crate::db::blobs; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; + +use super::paths::Names; + +/// Keywords with a Maildir flag letter of their own. +const FLAG_LETTERS: [(&str, char); 6] = [ + ("$draft", 'D'), + ("$flagged", 'F'), + ("$forwarded", 'P'), + ("$answered", 'R'), + ("$seen", 'S'), + ("$deleted", 'T'), +]; + +/// Dovecot gives each folder 26 keyword letters, `a` to `z`. +const MAX_CUSTOM_KEYWORDS: usize = 26; + +struct Mailbox { + id: i64, + name: String, + parent_id: Option, + role: Option, + is_subscribed: bool, +} + +struct Folder { + /// The Maildir++ name, `Parent.Child`; empty for the inbox. + maildir_name: String, + /// The folder's real name and those of its parents. + display_path: Vec, + dir: PathBuf, + role: Option, + is_subscribed: bool, + messages: u64, + custom_keywords: Vec, + keywords_dropped: u64, +} + +pub struct MailOutcome { + pub mailbox: TypeCounts, + pub email: TypeCounts, + pub manifest: Value, +} + +pub fn write( + conn: &Connection, + root: &Path, + dry_run: bool, + logger: Logger, +) -> Result { + let mailboxes = read_mailboxes(conn)?; + let mut folders = plan_folders(&mailboxes, root); + let mut mailbox_counts = TypeCounts::default(); + if !dry_run { + for folder in folders.values() { + make_maildir(&folder.dir, !folder.maildir_name.is_empty())?; + } + } + mailbox_counts.created = mailboxes.len() as u64; + + let root_folder = folders + .iter() + .find(|(_, f)| f.maildir_name.is_empty()) + .map(|(id, _)| *id) + .expect("plan_folders always makes a root"); + let mut email_counts = TypeCounts::default(); + let mut stmt = conn.prepare( + "SELECT id, blob_id, received_at, mailbox_ids, keywords FROM emails ORDER BY id", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + })?; + for row in rows { + let (id, blob_id, received_at, mailbox_ids, keywords) = row?; + let Some(bytes) = blobs::blob_bytes(conn, blob_id)? else { + logger.warn(&format!( + "email {id}: its message is missing from the archive" + )); + email_counts.failed += 1; + continue; + }; + let mut targets: Vec = serde_json::from_str::>(&mailbox_ids) + .unwrap_or_default() + .into_iter() + .filter(|m| folders.contains_key(m)) + .collect(); + if targets.is_empty() { + logger.warn(&format!( + "email {id}: in no folder that was written; put in the inbox" + )); + targets.push(root_folder); + } + let keywords: Vec = serde_json::from_str(&keywords).unwrap_or_default(); + let secs = delivery_secs(&received_at, &bytes); + for mailbox_id in targets { + let folder = folders.get_mut(&mailbox_id).expect("filtered above"); + let flags = flags_for(&keywords, folder, logger); + let name = format!("{secs}.M{id}.inbuxa-migrate,S={}:2,{flags}", bytes.len()); + if !dry_run { + write_message(&folder.dir.join("cur").join(name), &bytes, secs)?; + } + folder.messages += 1; + } + email_counts.created += 1; + } + + let mut entries: Vec<&Folder> = folders.values().collect(); + entries.sort_by(|a, b| a.maildir_name.cmp(&b.maildir_name)); + if !dry_run { + for folder in &entries { + write_keywords_file(folder)?; + } + } + let manifest = json!({ + "layout": "maildir++", + "path": "mail", + "folders": entries.iter().map(|f| folder_manifest(f, root)).collect::>(), + }); + Ok(MailOutcome { + mailbox: mailbox_counts, + email: email_counts, + manifest, + }) +} + +fn read_mailboxes(conn: &Connection) -> Result, Error> { + let mut stmt = conn.prepare( + "SELECT id, name, parent_id, role, is_subscribed FROM mailboxes ORDER BY sort_order, id", + )?; + let rows = stmt.query_map([], |row| { + Ok(Mailbox { + id: row.get(0)?, + name: row.get(1)?, + parent_id: row.get(2)?, + role: row.get(3)?, + is_subscribed: row.get::<_, i64>(4)? != 0, + }) + })?; + Ok(rows.collect::>()?) +} + +/// Every mailbox's directory, parents before children so a renamed parent +/// takes its children with it. +fn plan_folders(mailboxes: &[Mailbox], root: &Path) -> HashMap { + let by_id: HashMap = mailboxes.iter().map(|m| (m.id, m)).collect(); + let inbox = mailboxes + .iter() + .find(|m| m.role.as_deref() == Some("inbox")) + .map(|m| m.id); + let mut names = Names::default(); + names.reserve("INBOX"); + let mut folders: HashMap = HashMap::new(); + let mut order: Vec<&Mailbox> = mailboxes.iter().collect(); + order.sort_by_key(|m| depth(m, &by_id)); + for mailbox in order { + let parent = mailbox + .parent_id + .filter(|p| by_id.contains_key(p) && *p != mailbox.id); + let (maildir_name, display_path) = if Some(mailbox.id) == inbox { + (String::new(), vec![mailbox.name.clone()]) + } else { + let (prefix, mut display) = match parent.and_then(|p| folders.get(&p)) { + Some(p) if p.maildir_name.is_empty() => { + ("INBOX.".to_owned(), p.display_path.clone()) + } + Some(p) => (format!("{}.", p.maildir_name), p.display_path.clone()), + None => (String::new(), Vec::new()), + }; + display.push(mailbox.name.clone()); + let wanted = format!("{prefix}{}", encode_segment(&mailbox.name)); + (names.unique(&wanted, ""), display) + }; + let dir = if maildir_name.is_empty() { + root.to_path_buf() + } else { + root.join(format!(".{maildir_name}")) + }; + folders.insert( + mailbox.id, + Folder { + maildir_name, + display_path, + dir, + role: mailbox.role.clone(), + is_subscribed: mailbox.is_subscribed, + messages: 0, + custom_keywords: Vec::new(), + keywords_dropped: 0, + }, + ); + } + if !folders.values().any(|f| f.maildir_name.is_empty()) { + // Maildir++ needs its root to be a maildir, even with no inbox. + folders.insert( + i64::MIN, + Folder { + maildir_name: String::new(), + display_path: vec!["INBOX".to_owned()], + dir: root.to_path_buf(), + role: Some("inbox".to_owned()), + is_subscribed: true, + messages: 0, + custom_keywords: Vec::new(), + keywords_dropped: 0, + }, + ); + } + folders +} + +fn depth(mailbox: &Mailbox, by_id: &HashMap) -> usize { + let mut depth = 0; + let mut current = mailbox.parent_id; + while let Some(id) = current { + depth += 1; + if depth > by_id.len() { + break; + } + current = by_id.get(&id).and_then(|m| m.parent_id); + } + depth +} + +/// One level of a Maildir++ name: modified UTF-7, with the hierarchy +/// separator and anything a directory name can't hold turned into `_`. The +/// manifest keeps the real name. +fn encode_segment(name: &str) -> String { + let encoded: String = utf7::IMAP + .encode(name) + .chars() + .map(|c| match c { + '.' | '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_', + c => c, + }) + .collect(); + let trimmed = encoded.trim_end_matches(' '); + if trimmed.is_empty() { + "_".to_owned() + } else { + trimmed.to_owned() + } +} + +fn flags_for(keywords: &[String], folder: &mut Folder, logger: Logger) -> String { + let mut letters: Vec = Vec::new(); + for keyword in keywords { + let lower = keyword.to_ascii_lowercase(); + if let Some((_, letter)) = FLAG_LETTERS.iter().find(|(k, _)| *k == lower) { + letters.push(*letter); + continue; + } + let index = match folder + .custom_keywords + .iter() + .position(|k| k.eq_ignore_ascii_case(keyword)) + { + Some(i) => i, + None if folder.custom_keywords.len() < MAX_CUSTOM_KEYWORDS => { + folder.custom_keywords.push(keyword.clone()); + folder.custom_keywords.len() - 1 + } + None => { + if folder.keywords_dropped == 0 { + logger.warn(&format!( + "folder {:?}: more than {MAX_CUSTOM_KEYWORDS} keywords; \ + the rest are left off its messages", + folder.display_path.join("/") + )); + } + folder.keywords_dropped += 1; + continue; + } + }; + letters.push((b'a' + index as u8) as char); + } + letters.sort_unstable(); + letters.dedup(); + letters.into_iter().collect() +} + +/// When the message arrived, as Unix seconds: the archive's received time, +/// else its Date header, else now. +fn delivery_secs(received_at: &str, message: &[u8]) -> i64 { + let parsed = OffsetDateTime::parse(received_at, &Rfc3339) + .or_else(|_| OffsetDateTime::parse(received_at, &Rfc2822)) + .map(|t| t.unix_timestamp()) + .ok() + .or_else(|| { + mail_parser::MessageParser::default() + .parse_headers(message) + .and_then(|m| m.date().map(|d| d.to_timestamp())) + }); + parsed + .filter(|s| *s > 0) + .unwrap_or_else(|| OffsetDateTime::now_utc().unix_timestamp()) +} + +fn make_maildir(dir: &Path, is_subfolder: bool) -> Result<(), Error> { + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(dir.join(sub))?; + } + if is_subfolder { + fs::File::create(dir.join("maildirfolder"))?; + } + Ok(()) +} + +fn write_message(path: &Path, bytes: &[u8], secs: i64) -> Result<(), Error> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?; + file.write_all(bytes)?; + if secs > 0 { + file.set_modified(UNIX_EPOCH + Duration::from_secs(secs as u64)) + .or_else(|_| file.set_modified(SystemTime::now()))?; + } + Ok(()) +} + +fn write_keywords_file(folder: &Folder) -> Result<(), Error> { + if folder.custom_keywords.is_empty() { + return Ok(()); + } + let mut text = String::new(); + for (i, keyword) in folder.custom_keywords.iter().enumerate() { + text.push_str(&format!("{i} {keyword}\n")); + } + fs::write(folder.dir.join("dovecot-keywords"), text)?; + Ok(()) +} + +fn folder_manifest(folder: &Folder, root: &Path) -> Value { + let dir = match folder.dir.strip_prefix(root) { + Ok(rel) if !rel.as_os_str().is_empty() => Path::new("mail").join(rel), + _ => PathBuf::from("mail"), + }; + let mut entry = json!({ + "name": folder.display_path.last().cloned().unwrap_or_default(), + "path": folder.display_path, + "dir": dir.to_string_lossy().replace('\\', "/"), + "role": folder.role, + "subscribed": folder.is_subscribed, + "messages": folder.messages, + }); + if folder.keywords_dropped > 0 { + entry["keywordsDropped"] = json!(folder.keywords_dropped); + } + entry +} + +#[cfg(test)] +mod tests { + use super::*; + + fn mailbox(id: i64, name: &str, parent_id: Option, role: Option<&str>) -> Mailbox { + Mailbox { + id, + name: name.to_owned(), + parent_id, + role: role.map(str::to_owned), + is_subscribed: true, + } + } + + #[test] + fn folders_follow_maildir_plus_plus() { + let root = Path::new("/x"); + let folders = plan_folders( + &[ + mailbox(1, "Inbox", None, Some("inbox")), + mailbox(2, "Clients", Some(1), None), + mailbox(3, "Archive", None, Some("archive")), + mailbox(4, "2024", Some(3), None), + mailbox(5, "v1.0 notes", None, None), + mailbox(6, "Équipe & co", None, None), + mailbox(7, "INBOX", None, None), + ], + root, + ); + let name = |id| folders[&id].maildir_name.as_str(); + assert_eq!(name(1), ""); + assert_eq!(folders[&1].dir, root); + assert_eq!(name(2), "INBOX.Clients"); + assert_eq!(name(3), "Archive"); + assert_eq!(name(4), "Archive.2024"); + assert_eq!(folders[&4].dir, root.join(".Archive.2024")); + assert_eq!(name(5), "v1_0 notes"); + assert_eq!(folders[&5].display_path, vec!["v1.0 notes"]); + assert_eq!(name(6), "&AMk-quipe &- co"); + assert_eq!(name(7), "INBOX (2)"); + } + + #[test] + fn a_store_with_no_inbox_still_gets_a_root() { + let folders = plan_folders(&[mailbox(1, "Sent", None, Some("sent"))], Path::new("/x")); + assert!(folders.values().any(|f| f.maildir_name.is_empty())); + assert_eq!(folders[&1].maildir_name, "Sent"); + } + + #[test] + fn keywords_become_letters() { + let mut folder = Folder { + maildir_name: "Work".to_owned(), + display_path: vec!["Work".to_owned()], + dir: PathBuf::from("/x/.Work"), + role: None, + is_subscribed: true, + messages: 0, + custom_keywords: Vec::new(), + keywords_dropped: 0, + }; + let kw = |list: &[&str]| list.iter().map(|s| s.to_string()).collect::>(); + let logger = Logger::new(0); + assert_eq!( + flags_for( + &kw(&["$seen", "$flagged", "$answered"]), + &mut folder, + logger + ), + "FRS" + ); + assert_eq!( + flags_for(&kw(&["$Junk", "$seen"]), &mut folder, logger), + "Sa" + ); + assert_eq!( + flags_for(&kw(&["project-x", "$junk"]), &mut folder, logger), + "ab" + ); + assert_eq!(folder.custom_keywords, vec!["$Junk", "project-x"]); + + let many: Vec = (0..30).map(|i| format!("k{i}")).collect(); + let flags = flags_for(&many, &mut folder, logger); + // Two letters were already taken above. + assert_eq!(flags.len(), MAX_CUSTOM_KEYWORDS - 2); + assert_eq!(folder.keywords_dropped, 6); + } + + #[test] + fn delivery_time_prefers_the_archive_then_the_date_header() { + assert_eq!(delivery_secs("2024-03-01T12:00:00Z", b""), 1_709_294_400); + assert_eq!( + delivery_secs("Fri, 01 Mar 2024 12:00:00 +0000", b""), + 1_709_294_400 + ); + assert_eq!( + delivery_secs("", b"Date: Fri, 01 Mar 2024 12:00:00 +0000\r\n\r\nbody"), + 1_709_294_400 + ); + assert!(delivery_secs("", b"\r\n") > 1_709_294_400); + } +} diff --git a/src/export_files/mod.rs b/src/export_files/mod.rs new file mode 100644 index 0000000..0ad2ad1 --- /dev/null +++ b/src/export_files/mod.rs @@ -0,0 +1,419 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-files`: an archive written out as plain files in open formats, +//! so an account can leave for any server or app without this tool or any +//! vendor's store format. Mail is a Maildir++ tree, contacts are vCard, +//! calendars are iCalendar, filters are Sieve, files are files, and +//! `manifest.json` records what doesn't fit those formats. It never opens a +//! network connection. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{Duration, UNIX_EPOCH}; + +use rusqlite::Connection; +use serde_json::{Value, json}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::db::{blobs, init}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::{Summary, TypeCounts}; +use crate::types::ObjectType; + +pub mod cards; +pub mod mail; +pub mod paths; + +use paths::{Names, safe_component}; + +/// The layout this release writes; raised when `manifest.json` or the tree +/// changes in a way a reader has to know about. +pub const FORMAT_VERSION: u32 = 1; + +pub struct ExportFilesConfig { + pub archive: PathBuf, + pub dir: PathBuf, + pub objects: Option>, + pub dry_run: bool, + pub logger: Logger, +} + +pub fn run(config: ExportFilesConfig) -> Result { + if !config.archive.is_file() { + return Err(Error::Usage(format!( + "no archive at {}", + config.archive.display() + ))); + } + check_target(&config.dir)?; + let conn = init::open(&config.archive)?; + let wanted = |types: &[ObjectType]| match &config.objects { + None => true, + Some(list) => types.iter().any(|t| list.contains(t)), + }; + let dry_run = config.dry_run; + let logger = config.logger; + let dir = config.dir.as_path(); + if !dry_run { + make_private_dir(dir)?; + } + + let mut summary = Summary::default(); + let mut manifest = json!({ + "format": "inbuxa-migrate-files", + "formatVersion": FORMAT_VERSION, + "createdBy": format!("inbuxa-migrate {}", crate::version::shown()), + "created": OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(), + "sources": read_sources(&conn)?, + }); + + if wanted(&[ObjectType::Mailbox, ObjectType::Email]) { + let out = mail::write(&conn, &dir.join("mail"), dry_run, logger)?; + summary.per_type.push(("mailbox", out.mailbox)); + summary.per_type.push(("email", out.email)); + manifest["mail"] = out.manifest; + } + if wanted(&[ObjectType::AddressBook, ObjectType::ContactCard]) { + let out = cards::write_contacts(&conn, &dir.join("contacts"), dry_run, logger)?; + summary.per_type.push(("addressbook", out.collection)); + summary.per_type.push(("contactcard", out.item)); + manifest["contacts"] = out.manifest; + } + if wanted(&[ObjectType::Calendar, ObjectType::CalendarEvent]) { + let out = cards::write_calendars(&conn, &dir.join("calendars"), dry_run, logger)?; + summary.per_type.push(("calendar", out.collection)); + summary.per_type.push(("calendarevent", out.item)); + manifest["calendars"] = out.manifest; + } + if wanted(&[ObjectType::SieveScript]) { + let (counts, entries) = write_sieve(&conn, &dir.join("sieve"), dry_run, logger)?; + summary.per_type.push(("sievescript", counts)); + manifest["sieve"] = entries; + } + if wanted(&[ObjectType::FileNode]) { + let (counts, entries) = write_files(&conn, &dir.join("files"), dry_run, logger)?; + summary.per_type.push(("filenode", counts)); + manifest["files"] = entries; + } + if wanted(&[ObjectType::Identity]) { + let (counts, entries) = read_identities(&conn)?; + summary.per_type.push(("identity", counts)); + manifest["identities"] = entries; + } + if wanted(&[ObjectType::ParticipantIdentity]) { + let (counts, entries) = read_participant_identities(&conn)?; + summary.per_type.push(("participantidentity", counts)); + manifest["participantIdentities"] = entries; + } + + if dry_run { + logger.warn(&format!( + "dry run: nothing was written; {} would hold what is counted below", + dir.display() + )); + } else { + // Written last, so a tree with a manifest is a finished one. + let text = serde_json::to_string_pretty(&manifest).map_err(std::io::Error::other)?; + fs::write(dir.join("manifest.json"), text + "\n")?; + } + Ok(summary) +} + +/// Only a new or empty directory: what is written must be exactly the +/// archive, with nothing of an earlier run mixed in. +fn check_target(dir: &Path) -> Result<(), Error> { + match fs::read_dir(dir) { + Ok(mut entries) => { + if entries.next().is_some() { + return Err(Error::Usage(format!( + "{} is not empty; give a new or empty directory", + dir.display() + ))); + } + Ok(()) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotADirectory => Err(Error::Usage(format!( + "{} is not a directory", + dir.display() + ))), + Err(e) => Err(Error::Io(e)), + } +} + +/// Mail, contacts and calendars are as private as the archive they came +/// from. +fn make_private_dir(dir: &Path) -> Result<(), Error> { + fs::create_dir_all(dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(dir, fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} + +fn read_sources(conn: &Connection) -> Result { + let mut stmt = conn.prepare( + "SELECT kind, session_url, account_id, account_name, username FROM sources ORDER BY id", + )?; + let rows = stmt.query_map([], |r| { + Ok(json!({ + "kind": r.get::<_, String>(0)?, + "url": r.get::<_, String>(1)?, + "accountId": r.get::<_, String>(2)?, + "accountName": r.get::<_, Option>(3)?, + "username": r.get::<_, String>(4)?, + })) + })?; + Ok(Value::Array(rows.collect::>()?)) +} + +fn write_sieve( + conn: &Connection, + root: &Path, + dry_run: bool, + logger: Logger, +) -> Result<(TypeCounts, Value), Error> { + let mut counts = TypeCounts::default(); + let mut names = Names::default(); + let mut entries = Vec::new(); + let mut stmt = + conn.prepare("SELECT id, name, is_active, blob_id FROM sieve_scripts ORDER BY name, id")?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, Option>(1)?, + r.get::<_, i64>(2)? != 0, + r.get::<_, i64>(3)?, + )) + })?; + for row in rows { + let (id, name, is_active, blob_id) = row?; + let name = name.unwrap_or_else(|| format!("script-{id}")); + let Some(bytes) = blobs::blob_bytes(conn, blob_id)? else { + logger.warn(&format!( + "Sieve script {name:?}: its content is missing from the archive" + )); + counts.failed += 1; + continue; + }; + let file = names.unique(&safe_component(&name), ".sieve"); + if !dry_run { + fs::create_dir_all(root)?; + fs::write(root.join(&file), &bytes)?; + } + entries.push(json!({ + "name": name, + "file": format!("sieve/{file}"), + "active": is_active, + })); + counts.created += 1; + } + Ok((counts, Value::Array(entries))) +} + +struct FileNode { + id: i64, + parent_id: Option, + node_type: String, + blob_id: Option, + target: Option, + name: String, + media_type: Option, + modified: Option, + created: String, +} + +/// The file tree as directories and files. Links are listed in the manifest +/// and never made: the source chose their targets, and one could point +/// anywhere on this machine. +fn write_files( + conn: &Connection, + root: &Path, + dry_run: bool, + logger: Logger, +) -> Result<(TypeCounts, Value), Error> { + let mut stmt = conn.prepare( + "SELECT id, parent_id, node_type, blob_id, target, name, media_type, modified, created + FROM file_nodes ORDER BY name, id", + )?; + let nodes: Vec = stmt + .query_map([], |r| { + Ok(FileNode { + id: r.get(0)?, + parent_id: r.get(1)?, + node_type: r.get(2)?, + blob_id: r.get(3)?, + target: r.get(4)?, + name: r.get(5)?, + media_type: r.get(6)?, + modified: r.get(7)?, + created: r.get(8)?, + }) + })? + .collect::>()?; + let mut children: HashMap, Vec<&FileNode>> = HashMap::new(); + let known: std::collections::HashSet = nodes.iter().map(|n| n.id).collect(); + for node in &nodes { + let parent = node + .parent_id + .filter(|p| known.contains(p) && *p != node.id); + children.entry(parent).or_default().push(node); + } + let mut state = FilesState { + conn, + children: &children, + dry_run, + logger, + counts: TypeCounts::default(), + links: Vec::new(), + media_types: serde_json::Map::new(), + visited: std::collections::HashSet::new(), + }; + state.write_dir(None, root, "files")?; + let manifest = json!({ + "path": "files", + "mediaTypes": state.media_types, + "links": state.links, + }); + Ok((state.counts, manifest)) +} + +struct FilesState<'a> { + conn: &'a Connection, + children: &'a HashMap, Vec<&'a FileNode>>, + dry_run: bool, + logger: Logger, + counts: TypeCounts, + links: Vec, + media_types: serde_json::Map, + visited: std::collections::HashSet, +} + +impl FilesState<'_> { + fn write_dir(&mut self, parent: Option, dir: &Path, rel: &str) -> Result<(), Error> { + let Some(nodes) = self.children.get(&parent) else { + return Ok(()); + }; + let mut names = Names::default(); + for node in nodes.iter().copied() { + if !self.visited.insert(node.id) { + continue; + } + let name = names.unique(&safe_component(&node.name), ""); + let path = dir.join(&name); + let rel_path = format!("{rel}/{name}"); + match node.node_type.as_str() { + "directory" => { + if !self.dry_run { + fs::create_dir_all(&path)?; + } + self.counts.created += 1; + self.write_dir(Some(node.id), &path, &rel_path)?; + } + "file" => { + let bytes = match node.blob_id { + Some(blob) => blobs::blob_bytes(self.conn, blob)?, + None => Some(Vec::new()), + }; + let Some(bytes) = bytes else { + self.logger.warn(&format!( + "{rel_path}: its content is missing from the archive" + )); + self.counts.failed += 1; + continue; + }; + if !self.dry_run { + fs::create_dir_all(dir)?; + fs::write(&path, &bytes)?; + if let Some(when) = node + .modified + .as_deref() + .or(Some(node.created.as_str())) + .and_then(|t| OffsetDateTime::parse(t, &Rfc3339).ok()) + .filter(|t| t.unix_timestamp() > 0) + { + let _ = fs::File::options().write(true).open(&path).and_then(|f| { + f.set_modified( + UNIX_EPOCH + Duration::from_secs(when.unix_timestamp() as u64), + ) + }); + } + } + if let Some(media_type) = &node.media_type { + self.media_types + .insert(rel_path.clone(), Value::String(media_type.clone())); + } + self.counts.created += 1; + } + _ => { + let target = node + .target + .as_deref() + .and_then(|t| serde_json::from_str::(t).ok()); + self.links + .push(json!({ "path": rel_path, "target": target })); + self.counts.skipped += 1; + } + } + } + Ok(()) + } +} + +fn read_identities(conn: &Connection) -> Result<(TypeCounts, Value), Error> { + let mut stmt = conn.prepare( + "SELECT name, email, reply_to, bcc, text_signature, html_signature + FROM identities ORDER BY id", + )?; + let rows: Vec = stmt + .query_map([], |r| { + let json_column = |i: usize| -> rusqlite::Result { + Ok(r.get::<_, Option>(i)? + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or(Value::Null)) + }; + Ok(json!({ + "name": r.get::<_, String>(0)?, + "email": r.get::<_, String>(1)?, + "replyTo": json_column(2)?, + "bcc": json_column(3)?, + "textSignature": r.get::<_, String>(4)?, + "htmlSignature": r.get::<_, String>(5)?, + })) + })? + .collect::>()?; + let counts = TypeCounts { + created: rows.len() as u64, + ..Default::default() + }; + Ok((counts, Value::Array(rows))) +} + +fn read_participant_identities(conn: &Connection) -> Result<(TypeCounts, Value), Error> { + let mut stmt = conn.prepare( + "SELECT name, calendar_address, is_default FROM participant_identities ORDER BY id", + )?; + let rows: Vec = stmt + .query_map([], |r| { + Ok(json!({ + "name": r.get::<_, String>(0)?, + "calendarAddress": r.get::<_, String>(1)?, + "default": r.get::<_, i64>(2)? != 0, + })) + })? + .collect::>()?; + let counts = TypeCounts { + created: rows.len() as u64, + ..Default::default() + }; + Ok((counts, Value::Array(rows))) +} diff --git a/src/export_files/paths.rs b/src/export_files/paths.rs new file mode 100644 index 0000000..3d3e752 --- /dev/null +++ b/src/export_files/paths.rs @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! File names for what the archive holds. Every name comes from the source +//! server, so none is trusted as a path: each becomes one safe component, +//! and two that end up the same are told apart. + +use std::collections::HashSet; + +/// One path component made from a name the source chose: no separators, no +/// `.` or `..`, nothing Windows refuses, never empty. +pub fn safe_component(name: &str) -> String { + let mut out: String = name + .chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_', + c if c.is_control() => '_', + c => c, + }) + .collect(); + while out.ends_with(['.', ' ']) { + out.pop(); + } + let trimmed = out.trim_start(); + if trimmed.len() != out.len() { + out = trimmed.to_owned(); + } + if out.is_empty() || out.starts_with('.') { + out.insert(0, '_'); + } + if is_windows_reserved(&out) { + out.push('_'); + } + out +} + +fn is_windows_reserved(name: &str) -> bool { + let stem = name.split('.').next().unwrap_or(name).to_ascii_uppercase(); + matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && stem.as_bytes()[3].is_ascii_digit()) +} + +/// Names already used in one directory, compared without case so the tree +/// unpacks the same on a case-insensitive file system. +#[derive(Default)] +pub struct Names { + used: HashSet, +} + +impl Names { + pub fn reserve(&mut self, name: &str) { + self.used.insert(name.to_lowercase()); + } + + /// `stem` + `ext`, or `stem (2)` + `ext` and up when that is taken. + pub fn unique(&mut self, stem: &str, ext: &str) -> String { + let mut candidate = format!("{stem}{ext}"); + let mut n = 2; + while !self.used.insert(candidate.to_lowercase()) { + candidate = format!("{stem} ({n}){ext}"); + n += 1; + } + candidate + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn separators_and_traversal_never_survive() { + assert_eq!(safe_component("../../etc/passwd"), "_.._.._etc_passwd"); + assert_eq!(safe_component(".."), "_"); + assert_eq!(safe_component("."), "_"); + assert_eq!(safe_component(""), "_"); + assert_eq!(safe_component("a\\b"), "a_b"); + assert_eq!(safe_component("a\0b"), "a_b"); + assert_eq!(safe_component(".hidden"), "_.hidden"); + } + + #[test] + fn ordinary_names_are_kept() { + assert_eq!(safe_component("Family & Friends"), "Family & Friends"); + assert_eq!(safe_component("Équipe"), "Équipe"); + assert_eq!(safe_component("v1.0"), "v1.0"); + } + + #[test] + fn windows_trouble_is_avoided() { + assert_eq!(safe_component("CON"), "CON_"); + assert_eq!(safe_component("com1.txt"), "com1.txt_"); + assert_eq!(safe_component("Notes. "), "Notes"); + assert_eq!(safe_component("a:b?"), "a_b_"); + } + + #[test] + fn clashing_names_are_numbered_case_blind() { + let mut names = Names::default(); + assert_eq!(names.unique("Work", ".ics"), "Work.ics"); + assert_eq!(names.unique("work", ".ics"), "work (2).ics"); + assert_eq!(names.unique("Work", ".ics"), "Work (3).ics"); + names.reserve("INBOX"); + assert_eq!(names.unique("inbox", ""), "inbox (2)"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 4d0d2c5..d68c320 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ pub mod error; pub mod exchange; pub mod exchange_ews; pub mod exchange_graph; +pub mod export_files; pub mod imap; pub mod inspect; pub mod jmap; diff --git a/src/main.rs b/src/main.rs index bf90b65..a45972a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,8 +9,8 @@ use clap::Parser; use inbuxa_migrate::cli::{Action, Cli}; use inbuxa_migrate::error::Error; -use inbuxa_migrate::inspect; use inbuxa_migrate::sync::{self, RunOutcome, Summary}; +use inbuxa_migrate::{export_files, inspect}; fn main() { let code = run(); @@ -83,6 +83,10 @@ fn run() -> i32 { logger, ) } + Action::ExportFiles(config) => { + let logger = config.logger; + (RunOutcome::from_result(export_files::run(config)), logger) + } Action::Inspect(config) => { return match inspect::run(config) { Ok(()) => 0, diff --git a/src/sync/import_maildir/keywords.rs b/src/sync/import_maildir/keywords.rs index 5afb158..adc041d 100644 --- a/src/sync/import_maildir/keywords.rs +++ b/src/sync/import_maildir/keywords.rs @@ -1,9 +1,12 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::path::Path; + use mail_parser::mailbox::maildir::Flag; #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -37,6 +40,53 @@ pub fn translate_flags(flags: &[Flag], include_deleted: bool) -> Translation { } } +/// The translation with a message's own keywords added: those Dovecot +/// keeps as the letters `a` to `z`. +pub fn with_custom(mut translation: Translation, custom: &[String]) -> Translation { + for k in custom { + push_unique(&mut translation.keywords, k); + } + translation.keywords.sort(); + translation +} + +/// What the keyword letters `a` to `z` stand for in one folder, from its +/// `dovecot-keywords` file: a line `0 $Junk` names `a`, `1 work` names `b`. +pub fn read_keyword_names(folder_path: &Path) -> Vec> { + let mut names = vec![None; 26]; + let Ok(text) = std::fs::read_to_string(folder_path.join("dovecot-keywords")) else { + return names; + }; + for line in text.lines() { + let Some((index, name)) = line.split_once(' ') else { + continue; + }; + let name = name.trim(); + if let Ok(i) = index.parse::() + && i < names.len() + && !name.is_empty() + { + names[i] = Some(name.to_owned()); + } + } + names +} + +pub fn custom_keywords_from_filename(filename: &str, names: &[Option]) -> Vec { + let Some(part) = filename.rsplit_once("2,").map(|(_, s)| s) else { + return Vec::new(); + }; + let mut out = Vec::new(); + for ch in part.bytes().take_while(u8::is_ascii_alphanumeric) { + if ch.is_ascii_lowercase() + && let Some(Some(name)) = names.get((ch - b'a') as usize) + { + push_unique(&mut out, name); + } + } + out +} + fn push_unique(out: &mut Vec, k: &str) { if !out.iter().any(|x| x == k) { out.push(k.to_owned()); @@ -77,6 +127,32 @@ pub fn unique_id_from_filename(filename: &str) -> &str { mod tests { use super::*; + #[test] + fn dovecot_keyword_letters_name_keywords() { + let td = tempfile::tempdir().unwrap(); + std::fs::write( + td.path().join("dovecot-keywords"), + "0 $Junk\n1 project-x\nbad\n30 x\n", + ) + .unwrap(); + let names = read_keyword_names(td.path()); + assert_eq!(names[0].as_deref(), Some("$Junk")); + assert_eq!(names[1].as_deref(), Some("project-x")); + assert_eq!( + custom_keywords_from_filename("1.M1.h,S=10:2,FSab", &names), + vec!["$Junk", "project-x"] + ); + assert!(custom_keywords_from_filename("1.M1.h:2,Sc", &names).is_empty()); + let t = with_custom(translate_flags(&[Flag::Seen], false), &["$Junk".to_owned()]); + assert_eq!(t.keywords, vec!["$Junk", "$seen"]); + } + + #[test] + fn a_folder_without_dovecot_keywords_names_none() { + let td = tempfile::tempdir().unwrap(); + assert!(read_keyword_names(td.path()).iter().all(Option::is_none)); + } + #[test] fn empty_flags_yields_empty_keywords() { let t = translate_flags(&[], false); diff --git a/src/sync/import_maildir/messages.rs b/src/sync/import_maildir/messages.rs index 2038391..21bb0cf 100644 --- a/src/sync/import_maildir/messages.rs +++ b/src/sync/import_maildir/messages.rs @@ -20,7 +20,10 @@ use crate::sync::TypeCounts; use crate::sync::emailmeta::email_meta_from_blob; use crate::sync::keys::index_to_json; -use super::keywords::{Translation, flags_from_filename, translate_flags, unique_id_from_filename}; +use super::keywords::{ + Translation, custom_keywords_from_filename, flags_from_filename, read_keyword_names, + translate_flags, unique_id_from_filename, with_custom, +}; #[derive(Debug, Clone)] pub struct DiskEntry { @@ -28,6 +31,9 @@ pub struct DiskEntry { pub filename: String, pub path: PathBuf, pub flags: Vec, + /// Keywords beyond the standard flags, from the folder's + /// `dovecot-keywords`. + pub keywords: Vec, pub mtime_unix: u64, } @@ -40,6 +46,7 @@ pub struct DiskListing { pub fn list_folder(folder_path: &Path) -> std::io::Result { let mut out = DiskListing::default(); let mut seen: std::collections::HashMap = std::collections::HashMap::new(); + let keyword_names = read_keyword_names(folder_path); for sub in ["cur", "new"] { let dir = folder_path.join(sub); if !dir.is_dir() { @@ -80,6 +87,7 @@ pub fn list_folder(folder_path: &Path) -> std::io::Result { continue; } let flags = flags_from_filename(&filename); + let keywords = custom_keywords_from_filename(&filename, &keyword_names); let mtime_unix = match entry.metadata().and_then(|m| m.modified()).and_then(|t| { t.duration_since(UNIX_EPOCH).map_err(|e| { std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()) @@ -94,6 +102,7 @@ pub fn list_folder(folder_path: &Path) -> std::io::Result { filename, path, flags, + keywords, mtime_unix, }); } @@ -146,7 +155,10 @@ pub fn insert_new( entry: &DiskEntry, ) -> Result, InsertError> { let bytes = std::fs::read(&entry.path).map_err(InsertError::Io)?; - let translation = translate_flags(&entry.flags, ctx.include_deleted); + let translation = with_custom( + translate_flags(&entry.flags, ctx.include_deleted), + &entry.keywords, + ); if translation.has_trashed_flag && !ctx.include_deleted { return Ok(None); } @@ -219,7 +231,10 @@ pub fn refresh_flags( stored_keywords: &str, include_deleted: bool, ) -> Result { - let translation = translate_flags(&entry.flags, include_deleted); + let translation = with_custom( + translate_flags(&entry.flags, include_deleted), + &entry.keywords, + ); if translation.has_trashed_flag && !include_deleted { return Ok(PresentOutcome::DeleteRequested); } @@ -554,6 +569,7 @@ mod tests { filename: "a".into(), path: PathBuf::new(), flags: vec![], + keywords: vec![], mtime_unix: 0, }, DiskEntry { @@ -561,6 +577,7 @@ mod tests { filename: "b".into(), path: PathBuf::new(), flags: vec![], + keywords: vec![], mtime_unix: 0, }, ]; diff --git a/src/sync/import_maildir/tree.rs b/src/sync/import_maildir/tree.rs index d64515f..2df8f63 100644 --- a/src/sync/import_maildir/tree.rs +++ b/src/sync/import_maildir/tree.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -104,7 +105,7 @@ pub fn discover(root: &Path, automap_enabled: bool) -> Result Option { name.rfind('.').map(|i| name[..i].to_owned()) } +/// A folder's name as shown: Maildir++ keeps names in modified UTF-7, as +/// Dovecot and Courier write them. A name that isn't valid modified UTF-7 is +/// shown as it is on disk. +fn display_leaf(raw: &str) -> String { + crate::imap::name::decode_mailbox_name(raw).unwrap_or_else(|_| raw.to_owned()) +} + fn leaf_of(name: &str) -> &str { match name.rfind('.') { Some(i) => &name[i + 1..], @@ -179,7 +187,7 @@ pub fn restore_ephemeral_parents(folders: &mut Vec, root: &Path) if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) { continue; } - let leaf = leaf_of(&ancestor).to_owned(); + let leaf = display_leaf(leaf_of(&ancestor)); let parent_path = split_parent(&ancestor); to_add.push(ResolvedFolder { name: ancestor, @@ -378,6 +386,9 @@ mod tests { fn leaf_and_split_handle_root_and_nested() { assert_eq!(leaf_of("INBOX"), "INBOX"); assert_eq!(leaf_of("Archive.2025"), "2025"); + assert_eq!(display_leaf("&AMk-quipe &- co"), "Équipe & co"); + assert_eq!(display_leaf("Plain"), "Plain"); + assert_eq!(display_leaf("Équipe"), "Équipe"); assert_eq!(split_parent("INBOX"), None); assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned())); } diff --git a/src/sync/import_takeout/calendar.rs b/src/sync/import_takeout/calendar.rs index bb6dd95..7c7f27e 100644 --- a/src/sync/import_takeout/calendar.rs +++ b/src/sync/import_takeout/calendar.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -233,8 +234,8 @@ fn merge_container_id( fn parse_calendar_metadata(text: &str) -> CalendarMetadata { let mut out = CalendarMetadata::default(); let mut inside_vevent = false; - for raw_line in text.lines() { - let line = raw_line.trim_end_matches('\r'); + for line in unfolded_lines(text) { + let line = line.as_str(); if line.starts_with("BEGIN:VEVENT") || line.starts_with("BEGIN:VTODO") || line.starts_with("BEGIN:VJOURNAL") @@ -253,9 +254,9 @@ fn parse_calendar_metadata(text: &str) -> CalendarMetadata { continue; } if let Some(rest) = line.strip_prefix("X-WR-CALNAME:") { - out.x_wr_calname = Some(rest.trim().to_owned()); + out.x_wr_calname = Some(unescape_text(rest.trim())); } else if let Some(rest) = line.strip_prefix("X-WR-CALDESC:") { - out.x_wr_caldesc = Some(rest.trim().to_owned()); + out.x_wr_caldesc = Some(unescape_text(rest.trim())); } else if let Some(rest) = line.strip_prefix("X-WR-TIMEZONE:") { out.x_wr_timezone = Some(rest.trim().to_owned()); } @@ -263,10 +264,51 @@ fn parse_calendar_metadata(text: &str) -> CalendarMetadata { out } +/// Content lines with RFC 5545 folding undone. +fn unfolded_lines(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for raw in text.lines() { + let line = raw.trim_end_matches('\r'); + match (line.strip_prefix([' ', '\t']), out.last_mut()) { + (Some(rest), Some(last)) => last.push_str(rest), + _ => out.push(line.to_owned()), + } + } + out +} + +/// An iCalendar TEXT value with its `\\`, `\;`, `\,` and `\n` escapes undone. +fn unescape_text(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n' | 'N') => out.push('\n'), + Some(other) => out.push(other), + None => out.push('\\'), + } + } + out +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn parse_metadata_unfolds_and_unescapes() { + let meta = parse_calendar_metadata( + "BEGIN:VCALENDAR\r\nX-WR-CALNAME:Team\\, Berlin\\; o\r\n ps\r\n\ + X-WR-CALDESC:one\\ntwo\\\\three\r\nEND:VCALENDAR\r\n", + ); + assert_eq!(meta.x_wr_calname.as_deref(), Some("Team, Berlin; ops")); + assert_eq!(meta.x_wr_caldesc.as_deref(), Some("one\ntwo\\three")); + } + #[test] fn parse_metadata_finds_xwr_fields() { let ical = "BEGIN:VCALENDAR\r\n\ diff --git a/tests/export_files.rs b/tests/export_files.rs new file mode 100644 index 0000000..70e377f --- /dev/null +++ b/tests/export_files.rs @@ -0,0 +1,478 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-files`: an archive written out as open formats, and those files +//! read back by `import maildir` and `import takeout` with nothing lost +//! that the formats can hold. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use inbuxa_migrate::db::{blobs, init}; +use inbuxa_migrate::export_files::{ExportFilesConfig, run}; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::CommonConfig; +use inbuxa_migrate::sync::import_dav::calcard; +use inbuxa_migrate::sync::import_maildir::{self, MaildirImportConfig}; +use inbuxa_migrate::sync::import_takeout::{self, TakeoutImportConfig}; +use rusqlite::{Connection, params}; +use serde_json::Value; +use tempfile::TempDir; + +const VCARD_ALICE: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-alice\r\nFN:Alice Smith\r\n\ +EMAIL:alice@example.com\r\nEND:VCARD\r\n"; +const VCARD_BOB: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-bob\r\nFN:Bob Jones\r\n\ +TEL:+1 555 0100\r\nEND:VCARD\r\n"; +const EVENT: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\nBEGIN:VEVENT\r\nUID:ev-standup\r\n\ +DTSTAMP:20240101T000000Z\r\nDTSTART;TZID=Europe/Berlin:20240301T090000\r\nDURATION:PT15M\r\n\ +RRULE:FREQ=WEEKLY;COUNT=4\r\nSUMMARY:Standup\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; + +fn message(subject: &str) -> Vec { + format!( + "From: dave@example.net\r\nTo: alice@example.org\r\nSubject: {subject}\r\n\ + Message-ID: <{}@example.net>\r\nDate: Fri, 01 Mar 2024 12:00:00 +0000\r\n\r\n{subject}\r\n", + subject.replace(' ', "-") + ) + .into_bytes() +} + +/// An archive as an import would leave it: mail in nested and non-ASCII +/// folders, a message in two folders, standard and custom keywords, two +/// address books sharing a card, a calendar, Sieve scripts, files and a link. +fn seed(path: &Path) { + let conn = init::open(path).unwrap(); + conn.execute( + "INSERT INTO sources (id, kind, session_url, account_id, account_name, username) + VALUES (1, 'jmap', 'https://mail.example.org/jmap/session', 'a1', 'alice', 'alice')", + [], + ) + .unwrap(); + for (id, name, parent, role) in [ + (1, "Inbox", None, Some("inbox")), + (2, "Clients", Some(1), None), + (3, "Archive", None, Some("archive")), + (4, "2024", Some(3), None), + (5, "Équipe & co", None, None), + (6, "v1.0 notes", None, None), + ] { + conn.execute( + "INSERT INTO mailboxes (id, name, parent_id, role) VALUES (?1, ?2, ?3, ?4)", + params![id, name, parent, role], + ) + .unwrap(); + } + for (subject, mailboxes, keywords) in [ + ("Seen and flagged", "[1]", r#"["$seen","$flagged"]"#), + ("Junk in two folders", "[1,3]", r#"["$Junk"]"#), + ("Client note", "[2]", r#"["$answered","project-x"]"#), + ("Old report", "[4]", "[]"), + ("Bonjour", "[5]", r#"["$seen"]"#), + ("Release notes", "[6]", "[]"), + ] { + let blob = blobs::intern_blob(&conn, &message(subject)).unwrap(); + conn.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-03-01T12:00:00Z', ?2, ?3)", + params![blob, mailboxes, keywords], + ) + .unwrap(); + } + + conn.execute( + "INSERT INTO address_books (id, name, is_default) VALUES (1, 'Personal', 1), (2, 'Work', 0)", + [], + ) + .unwrap(); + for (vcard, books) in [(VCARD_ALICE, "[1,2]"), (VCARD_BOB, "[2]")] { + let card = calcard::vcard_to_jscontact(vcard, "/x").unwrap(); + conn.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![card.uid, books, card.data.to_string()], + ) + .unwrap(); + } + + conn.execute( + "INSERT INTO calendars (id, name, description, color, time_zone, is_default) + VALUES (1, 'Team, Berlin', 'Stand-ups', '#3366ff', 'Europe/Berlin', 1)", + [], + ) + .unwrap(); + let mut entry = calcard::ical_to_jscalendar_entries(EVENT) + .unwrap() + .remove(0); + calcard::strip_extracted_fields_from_event(&mut entry.data); + conn.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES ('[1]', ?1)", + params![entry.data.to_string()], + ) + .unwrap(); + + for (name, active, script) in [ + ( + "filters", + 1, + "require \"fileinto\";\r\nif header :contains \"subject\" \"report\" { fileinto \"Archive\"; }\r\n", + ), + ("old", 0, "keep;\r\n"), + ] { + let blob = blobs::intern_blob(&conn, script.as_bytes()).unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)", + params![name, active, blob], + ) + .unwrap(); + } + + let file_blob = blobs::intern_blob(&conn, b"hello\n").unwrap(); + conn.execute( + "INSERT INTO file_nodes (id, parent_id, node_type, blob_id, target, name, media_type, created) + VALUES (1, NULL, 'directory', NULL, NULL, 'Docs', NULL, '2024-01-01T00:00:00Z'), + (2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z'), + (3, NULL, 'symlink', NULL, '{\"path\":\"/etc/passwd\"}', 'link', NULL, '2024-01-01T00:00:00Z'), + (4, NULL, 'file', ?1, NULL, '../escape', NULL, '2024-01-01T00:00:00Z')", + params![file_blob], + ) + .unwrap(); + conn.execute( + "INSERT INTO identities (name, email) VALUES ('Alice Smith', 'alice@example.org')", + [], + ) + .unwrap(); +} + +struct Fixture { + _td: TempDir, + root: PathBuf, + archive: PathBuf, + out: PathBuf, +} + +fn exported() -> Fixture { + let td = tempfile::tempdir().unwrap(); + let root = td.path().to_path_buf(); + let archive = root.join("archive.sqlite"); + let out = root.join("out"); + seed(&archive); + let summary = run(ExportFilesConfig { + archive: archive.clone(), + dir: out.clone(), + objects: None, + dry_run: false, + logger: Logger::from_flags(true, 0), + }) + .unwrap(); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + Fixture { + _td: td, + root, + archive, + out, + } +} + +fn common(archive: &Path) -> CommonConfig { + CommonConfig { + archive: archive.to_path_buf(), + threads: 1, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + } +} + +fn manifest(out: &Path) -> Value { + serde_json::from_str(&fs::read_to_string(out.join("manifest.json")).unwrap()).unwrap() +} + +#[test] +fn writes_open_formats_and_a_manifest() { + let f = exported(); + let mail = f.out.join("mail"); + assert_eq!(fs::read_dir(mail.join("cur")).unwrap().count(), 2); + for dir in [ + ".INBOX.Clients", + ".Archive", + ".Archive.2024", + ".&AMk-quipe &- co", + ".v1_0 notes", + ] { + assert!(mail.join(dir).join("cur").is_dir(), "{dir}"); + assert!(mail.join(dir).join("maildirfolder").is_file(), "{dir}"); + } + assert_eq!( + fs::read_to_string(mail.join("dovecot-keywords")).unwrap(), + "0 $Junk\n" + ); + + let personal = fs::read_to_string(f.out.join("contacts/Personal.vcf")).unwrap(); + assert!(personal.contains("UID:card-alice") && !personal.contains("card-bob")); + let work = fs::read_to_string(f.out.join("contacts/Work.vcf")).unwrap(); + assert_eq!(work.matches("BEGIN:VCARD").count(), 2); + + let ics = fs::read_to_string(f.out.join("calendars/Team, Berlin.ics")).unwrap(); + assert!(ics.contains("X-WR-CALNAME:Team\\, Berlin\r\n"), "{ics}"); + assert!(ics.contains("UID:ev-standup"), "{ics}"); + assert_eq!(ics.matches("BEGIN:VCALENDAR").count(), 1); + + assert!(f.out.join("sieve/filters.sieve").is_file()); + assert_eq!( + fs::read(f.out.join("files/Docs/a.txt")).unwrap(), + b"hello\n" + ); + assert!(f.out.join("files/_.._escape").is_file()); + assert!(!f.out.join("files/link").exists()); + assert!(!f.root.join("escape").exists()); + + let m = manifest(&f.out); + assert_eq!(m["format"], "inbuxa-migrate-files"); + assert_eq!(m["formatVersion"], 1); + assert_eq!(m["sources"][0]["kind"], "jmap"); + let folders = m["mail"]["folders"].as_array().unwrap(); + let inbox = folders.iter().find(|x| x["role"] == "inbox").unwrap(); + assert_eq!(inbox["dir"], "mail"); + assert_eq!(inbox["name"], "Inbox"); + let equipe = folders + .iter() + .find(|x| x["dir"] == "mail/.&AMk-quipe &- co") + .unwrap(); + assert_eq!(equipe["name"], "Équipe & co"); + let dotted = folders + .iter() + .find(|x| x["dir"] == "mail/.v1_0 notes") + .unwrap(); + assert_eq!(dotted["name"], "v1.0 notes"); + let sieve = m["sieve"].as_array().unwrap(); + assert!( + sieve + .iter() + .any(|s| s["name"] == "filters" && s["active"] == true) + ); + assert_eq!(m["files"]["links"][0]["path"], "files/link"); + assert_eq!(m["identities"][0]["email"], "alice@example.org"); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&f.out).unwrap().permissions().mode(); + assert_eq!(mode & 0o077, 0, "export readable by others: {mode:o}"); + } +} + +/// Each message as (folder path, subject) → keywords, from an archive. +fn mail_by_folder(conn: &Connection) -> BTreeMap<(String, String), BTreeSet> { + let mut paths: BTreeMap = BTreeMap::new(); + let mut stmt = conn + .prepare("SELECT id, name, parent_id, role FROM mailboxes") + .unwrap(); + let rows: Vec<(i64, String, Option, Option)> = stmt + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + let name_of = |id: i64| { + let (_, name, _, role) = rows.iter().find(|r| r.0 == id).unwrap(); + if role.as_deref() == Some("inbox") { + "INBOX".to_owned() + } else { + name.clone() + } + }; + for (id, _, _, _) in &rows { + let mut parts = vec![name_of(*id)]; + let mut parent = rows.iter().find(|r| r.0 == *id).unwrap().2; + while let Some(p) = parent { + parts.insert(0, name_of(p)); + parent = rows.iter().find(|r| r.0 == p).unwrap().2; + } + paths.insert(*id, parts.join("/")); + } + let mut out = BTreeMap::new(); + let mut stmt = conn + .prepare("SELECT b.data, e.mailbox_ids, e.keywords FROM emails e JOIN blobs b ON b.id = e.blob_id") + .unwrap(); + let rows = stmt + .query_map([], |r| { + Ok(( + r.get::<_, Vec>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + }) + .unwrap(); + for row in rows { + let (data, mailboxes, keywords) = row.unwrap(); + let text = String::from_utf8(data).unwrap(); + let subject = text + .lines() + .find_map(|l| l.strip_prefix("Subject: ")) + .unwrap() + .to_owned(); + let keywords: BTreeSet = serde_json::from_str::>(&keywords) + .unwrap() + .into_iter() + .map(|k| k.to_lowercase()) + .collect(); + for mailbox in serde_json::from_str::>(&mailboxes).unwrap() { + out.insert((paths[&mailbox].clone(), subject.clone()), keywords.clone()); + } + } + out +} + +#[test] +fn mail_reads_back_with_import_maildir() { + let f = exported(); + let back = f.root.join("back.sqlite"); + import_maildir::run( + common(&back), + MaildirImportConfig { + maildir: f.out.join("mail"), + include: Vec::new(), + exclude: Vec::new(), + folder: Vec::new(), + automap: true, + include_deleted: false, + allow_source_change: false, + }, + ) + .unwrap(); + + let original = mail_by_folder(&Connection::open(&f.archive).unwrap()); + let read_back = mail_by_folder(&Connection::open(&back).unwrap()); + // Maildir++ has no `.` inside a folder name; the manifest keeps it. + let expected: BTreeMap<_, _> = original + .into_iter() + .map(|((path, subject), keywords)| ((path.replace("v1.0", "v1_0"), subject), keywords)) + .collect(); + assert_eq!(read_back, expected); + + let conn = Connection::open(&back).unwrap(); + let received: String = conn + .query_row("SELECT received_at FROM emails LIMIT 1", [], |r| r.get(0)) + .unwrap(); + assert_eq!(received, "2024-03-01T12:00:00Z"); +} + +#[test] +fn contacts_and_calendars_read_back_with_import_takeout() { + let f = exported(); + let back = f.root.join("back.sqlite"); + import_takeout::run( + common(&back), + TakeoutImportConfig { + takeout_root: f.out.clone(), + allow_source_change: false, + automap: true, + }, + ) + .unwrap(); + let conn = Connection::open(&back).unwrap(); + let uids: BTreeSet = conn + .prepare("SELECT uid FROM contact_cards") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + assert_eq!( + uids, + BTreeSet::from(["card-alice".to_owned(), "card-bob".to_owned()]) + ); + let calendar: String = conn + .query_row("SELECT name FROM calendars", [], |r| r.get(0)) + .unwrap(); + assert_eq!(calendar, "Team, Berlin"); + let event: String = conn + .query_row("SELECT data FROM calendar_events", [], |r| r.get(0)) + .unwrap(); + let event: Value = serde_json::from_str(&event).unwrap(); + assert_eq!(event["uid"], "ev-standup"); + assert_eq!(event["title"], "Standup"); + assert_eq!(event["timeZone"], "Europe/Berlin"); + assert_eq!(event["start"], "2024-03-01T09:00:00"); + assert_eq!(event["recurrenceRule"]["frequency"], "weekly"); +} + +#[test] +fn refuses_a_directory_that_is_not_empty() { + let td = tempfile::tempdir().unwrap(); + let archive = td.path().join("archive.sqlite"); + seed(&archive); + let out = td.path().join("out"); + fs::create_dir_all(&out).unwrap(); + fs::write(out.join("keep.txt"), "mine").unwrap(); + let err = run(ExportFilesConfig { + archive, + dir: out.clone(), + objects: None, + dry_run: false, + logger: Logger::from_flags(true, 0), + }) + .unwrap_err(); + assert_eq!(err.exit_code(), 1, "{err}"); + assert_eq!(fs::read_dir(&out).unwrap().count(), 1); +} + +#[test] +fn a_dry_run_counts_and_writes_nothing() { + let td = tempfile::tempdir().unwrap(); + let archive = td.path().join("archive.sqlite"); + seed(&archive); + let out = td.path().join("out"); + let summary = run(ExportFilesConfig { + archive, + dir: out.clone(), + objects: None, + dry_run: true, + logger: Logger::from_flags(true, 0), + }) + .unwrap(); + assert!(!out.exists()); + let email = summary + .per_type + .iter() + .find(|(t, _)| *t == "email") + .unwrap(); + assert_eq!(email.1.created, 6); +} + +#[test] +fn objects_limits_what_is_written() { + let td = tempfile::tempdir().unwrap(); + let archive = td.path().join("archive.sqlite"); + seed(&archive); + let out = td.path().join("out"); + run(ExportFilesConfig { + archive, + dir: out.clone(), + objects: Some(vec![inbuxa_migrate::types::ObjectType::SieveScript]), + dry_run: false, + logger: Logger::from_flags(true, 0), + }) + .unwrap(); + assert!(out.join("sieve").is_dir()); + assert!(!out.join("mail").exists()); + assert!(manifest(&out).get("mail").is_none()); +} + +#[test] +fn a_missing_archive_is_a_usage_error_and_creates_nothing() { + let td = tempfile::tempdir().unwrap(); + let archive = td.path().join("nope.sqlite"); + let err = run(ExportFilesConfig { + archive: archive.clone(), + dir: td.path().join("out"), + objects: None, + dry_run: false, + logger: Logger::from_flags(true, 0), + }) + .unwrap_err(); + assert_eq!(err.exit_code(), 1); + assert!(!archive.exists()); +} -- 2.54.0