From bb9bc66d63da93851765c141041f6b41c8f9e23a Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sat, 31 Oct 2026 13:41:06 -0700 Subject: [PATCH] import files: read an account export back, manifest and all export-files writes the layout specified at https://coffeylabs.org/specs/account-export/, and until now only its open formats could be read back (import maildir, import takeout). The parts only the manifest holds were lost on the way back in: real folder names, roles and subscriptions, the default address book and calendar, calendar colors and time zones, the active Sieve script, media types, links, identities and their signatures. import files reads all of it. Copies of a message kept in several folders come back as one message in each, matched by content, since the spec says nothing in a file name but the Maildir flags can be relied on. Every path in the manifest is checked to stay inside its part of the export and no symbolic link is followed. A tree without a manifest, or of a later layout version, is refused; the rc.2 format name is accepted. A second run brings the archive up to date and deletes nothing. Archives gain the source kind `files` (the sources table is rebuilt with foreign keys off, so nothing cascades) and a sync_id_files table. An archive exported, imported and exported again gives the same tree. --- CHANGELOG.md | 14 + README.md | 4 +- docs/usage.md | 46 +- src/cli.rs | 40 ++ src/db/files_ids.rs | 90 ++++ src/db/init.rs | 122 ++++- src/db/mod.rs | 1 + src/db/schema.sql | 13 +- src/main.rs | 4 + src/sync/import_files.rs | 402 ++++++++++++++ src/sync/import_files/cards.rs | 441 ++++++++++++++++ src/sync/import_files/mail.rs | 453 ++++++++++++++++ src/sync/import_files/manifest.rs | 317 +++++++++++ src/sync/import_files/other.rs | 532 +++++++++++++++++++ src/sync/import_files/paths.rs | 132 +++++ src/sync/import_takeout/contacts.rs | 3 +- src/sync/mod.rs | 1 + tests/import_files.rs | 789 ++++++++++++++++++++++++++++ 18 files changed, 3397 insertions(+), 7 deletions(-) create mode 100644 src/db/files_ids.rs create mode 100644 src/sync/import_files.rs create mode 100644 src/sync/import_files/cards.rs create mode 100644 src/sync/import_files/mail.rs create mode 100644 src/sync/import_files/manifest.rs create mode 100644 src/sync/import_files/other.rs create mode 100644 src/sync/import_files/paths.rs create mode 100644 tests/import_files.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5ab2a..c5c5049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Added +- `import files` reads an account export back into an archive: the tree + `export-files` writes, specified at + . Besides the mail, + contacts, calendars, Sieve scripts and files, it reads what only the + manifest holds: real folder names, roles and subscriptions, the default + address book and calendar, calendar colors and time zones, the active + script, media types, links and identities. Copies of a message in several + folders come back as one message in each. Paths in the manifest are + checked to stay inside the export, and no symbolic link is followed. An + archive gains the source kind `files` and a table for what it read at its + next import; until then `inspect` and the other read-only commands call it + an older layout. + ## [27.1-rc.8] The eighth release candidate of the 27.1 series: what a server needs to diff --git a/README.md b/README.md index c85a009..e44db02 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate inspect alice.sqlite what landed inbuxa-migrate export … alice.sqlite the archive into inbuxa inbuxa-migrate export-files alice.sqlite out the archive as plain files + inbuxa-migrate import files out alice.sqlite those files read back, manifest and all inbuxa-migrate export-imap … alice.sqlite its mail into any IMAP server inbuxa-migrate export-managesieve … alice.sqlite its filters, likewise inbuxa-migrate export-caldav … alice.sqlite its calendars into any CalDAV server @@ -48,7 +49,8 @@ server, `export-caldav` and `export-carddav` the calendars and contacts into any CalDAV and CardDAV server, and `export-files` is the way out to everything else: it writes the archive as a Maildir++ tree, vCard, iCalendar and Sieve files that other servers and mail, calendar and contacts apps open, with no network and no -store format in the way. For a whole server, `export-directory` writes the +store format in the way, and `import files` reads such a tree back whole. +For a whole server, `export-directory` writes the list of domains, accounts, groups, lists and aliases to set up again elsewhere, without passwords, and `import-directory` sets them up on an inbuxa server, with a new password for each user. `migrate` moves many diff --git a/docs/usage.md b/docs/usage.md index 0a63f44..47d2aef 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -215,6 +215,45 @@ Finds every `.mbox`, `.ics` and `.vcf` file under a directory and imports it. It is shaped for Google Takeout but reads any such tree. `--noautomap` stops it giving Gmail's system labels their mailbox roles. +### An account export + +``` +inbuxa-migrate import files +``` + +Reads an account export back: the directory `export-files` writes, laid out +as specifies, from this tool +or any other that writes the layout. No network. Where `import maildir` and +`import takeout` read the open formats on their own, this also reads the +manifest, so what those formats can't hold comes back too: + +- every folder's real name, its place in the tree, its role and whether it + was subscribed, in place of the names Maildir++ could give it on disk; +- which address book and calendar were the default, and each calendar's + description, color and time zone; +- which Sieve script was active; +- the files' media types and links; +- the identities with their reply-to, bcc and signatures, and the calendar + identities. + +A message kept in several folders is written to each folder's directory; +copies with the same content come back as one message in all of those +folders. The same goes for a card in two address books and an event in two +calendars. A message's arrival time is its file's modification time. + +The manifest's paths are data, so each is checked to stay inside its part of +the export, and nothing that is a symbolic link is followed: a link in the +tree is skipped with a warning, and the links an account had are made from +the manifest as archive entries. A directory without `manifest.json` was +interrupted, and is refused, as is a later version of the layout than this +release reads. 27.1.0-rc.2 named the format `inbuxa-migrate-files`; that is +read as the same format. + +Run again, it brings the archive up to date with the export: flags that +changed, a different active script, a renamed folder. It never deletes. +With `--dry-run` it checks the export and counts what it holds, and opens no +archive. + ### Microsoft Exchange (EWS) ``` @@ -484,7 +523,7 @@ DIR/ ``` - **Mail** is laid out as Dovecot and Courier keep it, so either can serve - it as it is, and `import maildir` reads it back. Folder names are in + it as it is. Folder names are in modified UTF-7. A `.` or `/` inside a folder's name becomes `_` on disk, since Maildir++ uses the dot between levels. A message kept in several folders is written to each. Read, answered, flagged, draft, forwarded and @@ -495,7 +534,6 @@ DIR/ files. **Calendars** are iCalendar, one VCALENDAR each, named by `X-WR-CALNAME`, with each time zone once. Photos and attachments the archive holds go inside as `data:` URIs, as `export` sends them. - `import takeout` reads both back. - **Sieve scripts** are written as they are; the manifest says which is active. - **Files** keep their names and dates. Links are listed in the manifest @@ -513,7 +551,9 @@ the way to finish it is to run again into an empty directory. The layout and the manifest are specified at , so other tools can read and -write them too. +write them too. `import files` reads the whole tree back, manifest and all +([An account export](#an-account-export)); `import maildir` and `import +takeout` read the mail, contacts and calendars without it. `--objects` takes the same types as `export`; naming a folder or a message type writes all of mail, and the same for contacts and calendars. With diff --git a/src/cli.rs b/src/cli.rs index 68a343f..151a34c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -30,6 +30,7 @@ use crate::sync::export_managesieve::ExportManageSieveConfig; use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig}; use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; +use crate::sync::import_files::FilesImportConfig; use crate::sync::import_imap::{ImapAuth, ImapImportConfig}; use crate::sync::import_maildir::MaildirImportConfig; use crate::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; @@ -126,6 +127,8 @@ enum Source { Maildir(Box), #[command(about = "Google Takeout (or any mbox/ics/vcf directory tree)")] Takeout(Box), + #[command(about = "An account export, as export-files writes it, manifest and all")] + Files(Box), #[command(about = "MS Exchange via EWS (SOAP/XML)", name = "exchange-ews")] ExchangeEws(Box), #[command( @@ -864,6 +867,7 @@ pub enum Action { ImportManageSieve(CommonConfig, ManageSieveImportConfig), ImportMaildir(CommonConfig, MaildirImportConfig), ImportTakeout(CommonConfig, TakeoutImportConfig), + ImportFiles(CommonConfig, FilesImportConfig), ImportExchangeEws(CommonConfig, EwsImportConfig), ImportExchangeGraph(CommonConfig, GraphImportConfig), Export(CommonConfig, ExportConfig), @@ -1042,6 +1046,7 @@ fn resolve_import(source: Source) -> Result { Source::Managesieve(args) => resolve_managesieve_import(*args), Source::Maildir(args) => resolve_maildir_import(*args), Source::Takeout(args) => resolve_takeout_import(*args), + Source::Files(args) => resolve_files_import(*args), Source::ExchangeEws(args) => resolve_exchange_ews_import(*args), Source::ExchangeGraph(args) => resolve_exchange_graph_import(*args), } @@ -1219,6 +1224,41 @@ fn resolve_takeout_import(args: TakeoutImportArgs) -> Result { )) } +#[derive(Args)] +pub struct FilesImportArgs { + #[arg( + value_name = "DIR", + help = "An account export: the directory export-files wrote, with its manifest.json" + )] + dir: PathBuf, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, + + #[arg( + long, + help = "Permit importing when the archive records an account export at a different path" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, +} + +fn resolve_files_import(args: FilesImportArgs) -> Result { + let common = common_config(&args.global, args.archive); + Ok(Action::ImportFiles( + common, + FilesImportConfig { + dir: args.dir, + allow_source_change: args.allow_source_change, + }, + )) +} + fn resolve_dav_import(args: DavImportArgs, kind: DavKindArg) -> Result { let common = common_config(&args.global, args.archive); let auth = resolve_dav_auth( diff --git a/src/db/files_ids.rs b/src/db/files_ids.rs new file mode 100644 index 0000000..20f9ad7 --- /dev/null +++ b/src/db/files_ids.rs @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! What `import files` has read from an account export, by a key of its +//! own for each thing: a folder's path of real names, a message's content +//! hash, a collection's file, a card's uid. Several keys may lead to one +//! archive row: a message kept in two folders, a card in two address books. + +use rusqlite::{Connection, OptionalExtension, params}; + +pub const MAILBOX: &str = "mailbox"; +pub const EMAIL: &str = "email"; +pub const ADDRESS_BOOK: &str = "addressbook"; +pub const CONTACT_CARD: &str = "contactcard"; +pub const CALENDAR: &str = "calendar"; +pub const CALENDAR_EVENT: &str = "calendarevent"; +pub const SIEVE_SCRIPT: &str = "sievescript"; +pub const FILE_NODE: &str = "filenode"; +pub const IDENTITY: &str = "identity"; +pub const PARTICIPANT_IDENTITY: &str = "participantidentity"; + +pub fn local_for( + conn: &Connection, + source_id: i64, + type_name: &str, + key: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_files + WHERE source_id = ?1 AND type_name = ?2 AND item_key = ?3", + params![source_id, type_name, key], + |row| row.get(0), + ) + .optional() +} + +/// Records `key` as `local_id`; a key already recorded is pointed at it. +pub fn record( + conn: &Connection, + source_id: i64, + type_name: &str, + key: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_files (source_id, type_name, item_key, local_id) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (source_id, type_name, item_key) DO UPDATE SET local_id = excluded.local_id", + params![source_id, type_name, key, local_id], + )?; + Ok(()) +} + +pub fn count(conn: &Connection, source_id: i64, type_name: &str) -> Result { + conn.query_row( + "SELECT count(DISTINCT local_id) FROM sync_id_files + WHERE source_id = ?1 AND type_name = ?2", + params![source_id, type_name], + |row| row.get::<_, i64>(0), + ) + .map(|n| n.max(0) as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + + #[test] + fn two_keys_may_lead_to_one_row_and_a_key_moves() { + let conn = Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + conn.execute( + "INSERT INTO sources (id, kind, session_url, account_id, username) + VALUES (1, 'files', 'file:///e', '/e', '')", + [], + ) + .unwrap(); + record(&conn, 1, EMAIL, "a", 7).unwrap(); + record(&conn, 1, EMAIL, "b", 7).unwrap(); + assert_eq!(local_for(&conn, 1, EMAIL, "a").unwrap(), Some(7)); + assert_eq!(count(&conn, 1, EMAIL).unwrap(), 1); + record(&conn, 1, EMAIL, "a", 8).unwrap(); + assert_eq!(local_for(&conn, 1, EMAIL, "a").unwrap(), Some(8)); + assert_eq!(local_for(&conn, 1, MAILBOX, "a").unwrap(), None); + } +} diff --git a/src/db/init.rs b/src/db/init.rs index f25ed96..ec6248b 100644 --- a/src/db/init.rs +++ b/src/db/init.rs @@ -5,7 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use rusqlite::{Connection, OpenFlags}; +use rusqlite::{Connection, OpenFlags, OptionalExtension}; pub const SCHEMA_SQL: &str = include_str!("schema.sql"); @@ -126,6 +126,9 @@ fn schema_tables() -> impl Iterator { } pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> { + // Before the transaction: it turns foreign keys off, which SQLite only + // allows outside one. + ensure_sources_accept_files(conn)?; let tx = conn.unchecked_transaction()?; tx.execute_batch(SCHEMA_SQL)?; ensure_calendar_events_data_type(&tx)?; @@ -245,6 +248,64 @@ fn ensure_dav_ids_share_local_rows(conn: &Connection) -> Result<(), OpenError> { Ok(()) } +/// `import files` records its source as the kind `files`, which the +/// `sources` table's CHECK didn't allow before. SQLite can't change a CHECK +/// in place, so an older archive's table is rebuilt. Foreign keys are off +/// while it is: with them on, dropping the old table would cascade into +/// every row that came from a source. +fn ensure_sources_accept_files(conn: &Connection) -> Result<(), OpenError> { + let sql: Option = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'sources'", + [], + |row| row.get(0), + ) + .optional()?; + // Only the table as earlier releases made it: its kind list ended at + // `takeout`. + let Some(sql) = sql else { return Ok(()) }; + if sql.contains("'files'") || !sql.contains("'takeout')") { + return Ok(()); + } + let foreign_keys: i64 = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?; + conn.pragma_update(None, "foreign_keys", "OFF")?; + let rebuilt = rebuild_sources(conn); + if foreign_keys != 0 { + conn.pragma_update(None, "foreign_keys", "ON")?; + } + rebuilt +} + +fn rebuild_sources(conn: &Connection) -> Result<(), OpenError> { + let tx = conn.unchecked_transaction()?; + tx.execute_batch( + "CREATE TABLE sources_new ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('jmap','imap','caldav','carddav','webdav','managesieve','maildir','exchange_ews','exchange_graph','takeout','files')), + session_url TEXT NOT NULL, + account_id TEXT NOT NULL, + account_name TEXT, + username TEXT NOT NULL, + UNIQUE (kind, session_url, account_id) + ); + INSERT INTO sources_new (id, kind, session_url, account_id, account_name, username) + SELECT id, kind, session_url, account_id, account_name, username FROM sources; + DROP TABLE sources; + ALTER TABLE sources_new RENAME TO sources;", + )?; + // Every row that names a source still finds it. + let dangling = tx.prepare("PRAGMA foreign_key_check")?.exists([])?; + if dangling { + return Err(OpenError::NotAnArchive( + "rebuilding the archive's sources table left rows without a source; \ + nothing was changed" + .to_owned(), + )); + } + tx.commit()?; + Ok(()) +} + /// An archive holds a whole mailbox, so it is created readable by its owner /// only. SQLite gives its `-wal` and `-shm` files the database file's mode, /// so they follow. An existing archive others can read is left as it is, @@ -494,6 +555,65 @@ mod permission_tests { mod migration_tests { use super::*; + #[test] + fn an_older_sources_table_is_rebuilt_to_accept_files_and_keeps_every_row() { + let conn = Connection::open_in_memory().unwrap(); + // The table as releases before `import files` made it. + let old = SCHEMA_SQL.replace("'takeout','files')", "'takeout')"); + assert_ne!(old, SCHEMA_SQL); + conn.execute_batch(&old).unwrap(); + conn.pragma_update(None, "foreign_keys", "ON").unwrap(); + conn.execute_batch( + "INSERT INTO sources (id, kind, session_url, account_id, account_name, username) + VALUES (3, 'maildir', 'file:///m', '/m', 'm', ''); + INSERT INTO sync_id_maildir VALUES (3, 'mailbox', 'INBOX', 'x', 9);", + ) + .unwrap(); + assert!( + conn.execute( + "INSERT INTO sources (kind, session_url, account_id, username) + VALUES ('files', 'file:///e', '/e', '')", + [], + ) + .is_err(), + "the old CHECK refuses it" + ); + + apply_schema(&conn).unwrap(); + + conn.execute( + "INSERT INTO sources (kind, session_url, account_id, username) + VALUES ('files', 'file:///e', '/e', '')", + [], + ) + .expect("files is a source kind now"); + let kept: (String, i64) = conn + .query_row( + "SELECT s.kind, m.local_id FROM sync_id_maildir m JOIN sources s ON s.id = m.source_id", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!( + kept, + ("maildir".to_owned(), 9), + "rows that came from it stayed" + ); + let on: i64 = conn + .query_row("PRAGMA foreign_keys", [], |r| r.get(0)) + .unwrap(); + assert_eq!(on, 1, "foreign keys are back on"); + // Deleting the source still cascades, so the reference survived. + conn.execute("DELETE FROM sources WHERE id = 3", []) + .unwrap(); + let left: i64 = conn + .query_row("SELECT count(*) FROM sync_id_maildir", [], |r| r.get(0)) + .unwrap(); + assert_eq!(left, 0); + // A second pass finds nothing to do. + apply_schema(&conn).unwrap(); + } + #[test] fn an_older_dav_map_is_rebuilt_to_share_local_rows() { let conn = Connection::open_in_memory().unwrap(); diff --git a/src/db/mod.rs b/src/db/mod.rs index 351d35a..96abfeb 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -10,6 +10,7 @@ pub mod dav_ids; pub mod defaults; pub mod exchange_ews_ids; pub mod exchange_graph_ids; +pub mod files_ids; pub mod ids; pub mod imap_ids; pub mod imap_state; diff --git a/src/db/schema.sql b/src/db/schema.sql index 6603294..cfc56de 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -1,6 +1,6 @@ CREATE TABLE IF NOT EXISTS sources ( id INTEGER PRIMARY KEY, - kind TEXT NOT NULL CHECK (kind IN ('jmap','imap','caldav','carddav','webdav','managesieve','maildir','exchange_ews','exchange_graph','takeout')), + kind TEXT NOT NULL CHECK (kind IN ('jmap','imap','caldav','carddav','webdav','managesieve','maildir','exchange_ews','exchange_graph','takeout','files')), session_url TEXT NOT NULL, account_id TEXT NOT NULL, account_name TEXT, @@ -268,3 +268,14 @@ CREATE TABLE IF NOT EXISTS sync_id_takeout ( CREATE INDEX IF NOT EXISTS sync_id_takeout_type_idx ON sync_id_takeout (source_id, type_name); + +CREATE TABLE IF NOT EXISTS sync_id_files ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL, + item_key TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, item_key) +); + +CREATE INDEX IF NOT EXISTS sync_id_files_local_idx + ON sync_id_files (source_id, type_name, local_id); diff --git a/src/main.rs b/src/main.rs index 8085124..5f94ff7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,6 +61,10 @@ fn run() -> i32 { let logger = common.logger; (sync::import_takeout::run_reporting(common, config), logger) } + Action::ImportFiles(common, config) => { + let logger = common.logger; + (sync::import_files::run_reporting(common, config), logger) + } Action::ImportExchangeEws(common, config) => { let logger = common.logger; ( diff --git a/src/sync/import_files.rs b/src/sync/import_files.rs new file mode 100644 index 0000000..480407f --- /dev/null +++ b/src/sync/import_files.rs @@ -0,0 +1,402 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `import files`: an account export (the layout `export-files` writes, +//! specified at ) read back +//! into an archive, manifest and all. `import maildir` and `import takeout` +//! read the open formats; this reads what only the manifest holds as well: +//! real folder names, roles and subscriptions, which address book and +//! calendar were the default, their colors and time zones, the active Sieve +//! script, media types and links, and the identities with their signatures. +//! +//! Paths in the manifest are data. Each one is checked to stay inside its +//! part of the export, and nothing that is a symbolic link is followed. A +//! run that is repeated brings the archive up to date with the export; it +//! never deletes. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; + +use rusqlite::{Connection, OptionalExtension, params}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use url::Url; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::import_dav::calcard; +use crate::sync::import_takeout::contacts::split_vcards; +use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; + +pub mod cards; +pub mod mail; +pub mod manifest; +pub mod other; +pub mod paths; + +use manifest::Manifest; + +pub const SOURCE_KIND: &str = "files"; + +#[derive(Debug)] +pub struct FilesImportConfig { + pub dir: PathBuf, + pub allow_source_change: bool, +} + +pub fn run(common: CommonConfig, config: FilesImportConfig) -> Result { + run_reporting(common, config).into_result() +} + +pub fn run_reporting(common: CommonConfig, config: FilesImportConfig) -> RunOutcome { + let mut summary = Summary::default(); + let error = run_into(common, config, &mut summary).err(); + RunOutcome { summary, error } +} + +fn run_into( + common: CommonConfig, + config: FilesImportConfig, + summary: &mut Summary, +) -> Result<(), Error> { + let logger = common.logger; + let root = fs::canonicalize(&config.dir) + .map_err(|e| Error::Usage(format!("{}: {e}", config.dir.display())))?; + if !root.is_dir() { + return Err(Error::Usage(format!("not a directory: {}", root.display()))); + } + let manifest = manifest::read(&root)?; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "account export {} written by {} at {}", + root.display(), + manifest + .created_by + .as_deref() + .unwrap_or("an unnamed program"), + manifest.created.as_deref().unwrap_or("an unrecorded time"), + ); + } + + if common.dry_run { + *summary = count_only(&root, &manifest, logger)?; + logger.warn("dry run: nothing was written; the counts are what the export holds"); + return Ok(()); + } + + let mut conn = db::init::open(&common.archive)?; + let session_url = Url::from_file_path(&root) + .map(|u| u.to_string()) + .map_err(|_| Error::Usage(format!("cannot name {} as a file: URL", root.display())))?; + let account_id = root.to_string_lossy().into_owned(); + if let Some((existing_url, _)) = + db::sources::conflicting_source(&conn, SOURCE_KIND, &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records the account export at {existing_url}; re-run with \ + --allow-source-change or use a fresh archive" + ))); + } + let original = manifest.sources.first(); + let account_name = original + .and_then(|s| s.account_name.clone().or_else(|| s.username.clone())) + .or_else(|| root.file_name().map(|n| n.to_string_lossy().into_owned())); + if logger.enabled(LEVEL_PROGRESS) + && let Some(source) = original + { + eprintln!( + "the account came from {} ({})", + source.url.as_deref().unwrap_or("an unrecorded address"), + source.kind.as_deref().unwrap_or("an unrecorded kind"), + ); + } + let source_id = db::sources::upsert_source( + &conn, + &SourceKey { + kind: SOURCE_KIND.to_owned(), + session_url, + account_id, + }, + account_name.as_deref(), + "", + )?; + + let result = import_parts(&mut conn, &root, source_id, &manifest, logger, summary); + if result.is_ok() && !summary.any_failed() { + let tx = conn.unchecked_transaction()?; + db::blobs::gc_orphan_blobs(&tx)?; + tx.commit()?; + } + if logger.enabled(LEVEL_DEFAULT) { + let line: Vec = summary + .per_type + .iter() + .map(|(name, c)| { + format!( + "{name} c={} u={} s={} f={}", + c.created, c.updated, c.skipped, c.failed + ) + }) + .collect(); + eprintln!("files: {}", line.join(" | ")); + } + result +} + +/// Each part the manifest lists, in turn; one that is missing wasn't +/// exported and is left as the archive has it. +fn import_parts( + conn: &mut Connection, + root: &Path, + source_id: i64, + manifest: &Manifest, + logger: Logger, + summary: &mut Summary, +) -> Result<(), Error> { + if let Some(mail) = &manifest.mail { + let out = mail::import(conn, root, source_id, mail, logger)?; + summary.per_type.push(("mailbox", out.mailbox)); + summary.per_type.push(("email", out.email)); + } + if let Some(books) = &manifest.contacts { + let out = cards::import_contacts(conn, root, source_id, books, logger)?; + summary.per_type.push(("addressbook", out.collection)); + summary.per_type.push(("contactcard", out.item)); + } + if let Some(calendars) = &manifest.calendars { + let out = cards::import_calendars(conn, root, source_id, calendars, logger)?; + summary.per_type.push(("calendar", out.collection)); + summary.per_type.push(("calendarevent", out.item)); + } + if let Some(scripts) = &manifest.sieve { + let counts = other::import_sieve(conn, root, source_id, scripts, logger)?; + summary.per_type.push(("sievescript", counts)); + } + if let Some(files) = &manifest.files { + let created = manifest + .created + .clone() + .filter(|c| OffsetDateTime::parse(c, &Rfc3339).is_ok()) + .unwrap_or_else(now_rfc3339); + let counts = other::import_files(conn, root, source_id, files, &created, logger)?; + summary.per_type.push(("filenode", counts)); + } + if let Some(identities) = &manifest.identities { + let counts = other::import_identities(conn, source_id, identities)?; + summary.per_type.push(("identity", counts)); + } + if let Some(identities) = &manifest.participant_identities { + let counts = other::import_participant_identities(conn, source_id, identities, logger)?; + summary.per_type.push(("participantidentity", counts)); + } + Ok(()) +} + +/// What a run would read, counted from the export without opening the +/// archive. A message kept in several folders counts once per folder. +fn count_only(root: &Path, manifest: &Manifest, logger: Logger) -> Result { + let found = |n: usize| TypeCounts { + fetched: n as u64, + ..Default::default() + }; + let mut summary = Summary::default(); + if let Some(mail) = &manifest.mail { + let mut messages = 0; + let mut failed = 0; + for folder in &mail.folders { + match paths::resolve(root, &folder.dir, &mail.path) { + Ok(dir) => { + for sub in ["cur", "new"] { + let sub_dir = dir.join(sub); + if paths::is_real_dir(&sub_dir) { + messages += fs::read_dir(&sub_dir)? + .filter_map(Result::ok) + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .filter(|e| !e.file_name().to_string_lossy().starts_with('.')) + .count(); + } + } + } + Err(reason) => { + logger.warn(&format!("folder {:?}: {reason}", folder.name)); + failed += 1; + } + } + } + summary + .per_type + .push(("mailbox", found(mail.folders.len()))); + let mut email = found(messages); + email.failed = failed; + summary.per_type.push(("email", email)); + } + for (part, books, collection, item) in [ + ("contacts", &manifest.contacts, "addressbook", "contactcard"), + ( + "calendars", + &manifest.calendars, + "calendar", + "calendarevent", + ), + ] { + let Some(books) = books else { continue }; + let mut items = TypeCounts::default(); + for book in books { + match cards::read_part_file(root, &book.file, part, logger) { + Some(text) if part == "contacts" => { + items.fetched += split_vcards(&text).len() as u64 + } + Some(text) => match calcard::ical_to_jscalendar_entries(&text) { + Ok(entries) => items.fetched += entries.len() as u64, + Err(calcard::CalcardError::NoEntries) => {} + Err(e) => { + logger.warn(&format!("{}: {e}", book.file)); + items.failed += 1; + } + }, + None => items.failed += 1, + } + } + summary.per_type.push((collection, found(books.len()))); + summary.per_type.push((item, items)); + } + if let Some(scripts) = &manifest.sieve { + let mut counts = TypeCounts::default(); + for script in scripts { + match paths::resolve(root, &script.file, "sieve") { + Ok(path) if paths::is_real_file(&path) => counts.fetched += 1, + Ok(_) => counts.failed += 1, + Err(reason) => { + logger.warn(&reason); + counts.failed += 1; + } + } + } + summary.per_type.push(("sievescript", counts)); + } + if let Some(files) = &manifest.files { + let base = root.join(&files.path); + let nodes = if paths::segments(&files.path).map(|s| s.len()) == Ok(1) + && paths::is_real_dir(&base) + { + count_tree(&base)? + } else { + 0 + }; + summary + .per_type + .push(("filenode", found(nodes + files.links.len()))); + } + if let Some(identities) = &manifest.identities { + summary.per_type.push(("identity", found(identities.len()))); + } + if let Some(identities) = &manifest.participant_identities { + summary + .per_type + .push(("participantidentity", found(identities.len()))); + } + Ok(summary) +} + +fn count_tree(dir: &Path) -> Result { + let mut n = 0; + for entry in fs::read_dir(dir)? { + let entry = entry?; + let file_type = entry.file_type()?; + if file_type.is_dir() { + n += 1 + count_tree(&entry.path())?; + } else if file_type.is_file() { + n += 1; + } + } + Ok(n) +} + +/// Rows a run found already in the archive, as they were before it changed +/// them, so the end of the run can count which it actually changed. +#[derive(Default)] +pub(crate) struct Rerun { + before: HashMap>, +} + +impl Rerun { + pub(crate) fn before( + &mut self, + conn: &Connection, + table: &str, + columns: &str, + id: i64, + ) -> Result<(), Error> { + if self.before.contains_key(&id) { + return Ok(()); + } + if let Some(row) = read_row(conn, table, columns, id)? { + self.before.insert(id, row); + } + Ok(()) + } + + pub(crate) fn tally( + &self, + conn: &Connection, + table: &str, + columns: &str, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + for (id, was) in &self.before { + match read_row(conn, table, columns, *id)? { + Some(now) if now == *was => counts.skipped += 1, + _ => counts.updated += 1, + } + } + Ok(()) + } +} + +fn read_row( + conn: &Connection, + table: &str, + columns: &str, + id: i64, +) -> Result>, Error> { + let mut stmt = conn.prepare(&format!("SELECT {columns} FROM {table} WHERE id = ?1"))?; + let n = stmt.column_count(); + Ok(stmt + .query_row(params![id], |r| { + (0..n) + .map(|i| r.get::<_, rusqlite::types::Value>(i)) + .collect() + }) + .optional()?) +} + +/// A file's modification time as RFC 3339, if it has a usable one. +pub(crate) fn mtime_rfc3339(meta: &fs::Metadata) -> Option { + let secs = meta + .modified() + .ok()? + .duration_since(UNIX_EPOCH) + .ok()? + .as_secs(); + if secs == 0 { + return None; + } + OffsetDateTime::from_unix_timestamp(i64::try_from(secs).ok()?) + .ok()? + .format(&Rfc3339) + .ok() +} + +pub(crate) fn now_rfc3339() -> String { + OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) +} diff --git a/src/sync/import_files/cards.rs b/src/sync/import_files/cards.rs new file mode 100644 index 0000000..1af442d --- /dev/null +++ b/src/sync/import_files/cards.rs @@ -0,0 +1,441 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Address books from `contacts/*.vcf` and calendars from +//! `calendars/*.ics`, each named and set up as the manifest says: its real +//! name, description, color, time zone and whether it was the default. A +//! card or event in two files is one row in both collections. + +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use rusqlite::{Connection, OptionalExtension, params}; +use serde_json::Value; + +use crate::db::{cards, defaults, files_ids}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::import_dav::calcard; +use crate::sync::import_takeout::contacts::split_vcards; +use crate::types::ObjectType; + +use super::mail::row_exists; +use super::manifest::Collection; +use super::{Rerun, paths}; + +pub struct Outcome { + pub collection: TypeCounts, + pub item: TypeCounts, +} + +const BOOK_COLUMNS: &str = "name, description, sort_order, is_default"; +const CARD_COLUMNS: &str = "address_book_ids, data"; +const CALENDAR_COLUMNS: &str = "name, description, color, time_zone, sort_order, is_default"; +const EVENT_COLUMNS: &str = "calendar_ids, is_draft, use_default_alerts, data, data_type"; + +pub fn import_contacts( + conn: &mut Connection, + root: &Path, + source_id: i64, + books: &[Collection], + logger: Logger, +) -> Result { + let mut out = Outcome { + collection: TypeCounts::default(), + item: TypeCounts::default(), + }; + let mut collections = Rerun::default(); + let mut items = Rerun::default(); + let mut seen: HashMap = HashMap::new(); + for (order, book) in books.iter().enumerate() { + let tx = conn.transaction()?; + let book_id = ensure_collection( + &tx, + source_id, + Kind::Contacts, + book, + order, + &mut collections, + &mut out.collection, + logger, + )?; + let Some(text) = read_part_file(root, &book.file, "contacts", logger) else { + out.item.failed += 1; + tx.commit()?; + continue; + }; + let chunks = split_vcards(&text); + for (ordinal, chunk) in chunks.iter().enumerate() { + let card = match calcard::vcard_to_jscontact(chunk, &format!("{}#{ordinal}", book.file)) + { + Ok(card) => card, + Err(e) => { + logger.warn(&format!("{} card {ordinal}: {e}", book.file)); + out.item.failed += 1; + continue; + } + }; + let data = card.data.to_string(); + if let Some(&id) = seen.get(&card.uid) { + // The same card in another address book. + add_to_collection(&tx, "contact_cards", "address_book_ids", id, book_id)?; + continue; + } + let ours = + match files_ids::local_for(&tx, source_id, files_ids::CONTACT_CARD, &card.uid)? { + Some(id) if row_exists(&tx, "contact_cards", id)? => Some(id), + _ => None, + }; + // The uid is the archive's key for a card, whoever put it there. + let theirs = match ours { + Some(_) => None, + None => tx + .query_row( + "SELECT id, address_book_ids FROM contact_cards WHERE uid = ?1", + params![card.uid], + |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)), + ) + .optional()?, + }; + let id = match (ours, theirs) { + (Some(id), _) => { + // Read on an earlier run: now in the books it is in now. + items.before(&tx, "contact_cards", CARD_COLUMNS, id)?; + tx.execute( + "UPDATE contact_cards SET address_book_ids = ?1, data = ?2 WHERE id = ?3", + params![format!("[{book_id}]"), data, id], + )?; + id + } + (None, Some((id, books))) => { + // Another import's card: it keeps the books it was in. + items.before(&tx, "contact_cards", CARD_COLUMNS, id)?; + tx.execute( + "UPDATE contact_cards SET address_book_ids = ?1, data = ?2 WHERE id = ?3", + params![cards::with_book(&books, book_id), data, id], + )?; + id + } + (None, None) => { + tx.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![card.uid, format!("[{book_id}]"), data], + )?; + out.item.created += 1; + out.item.fetched += 1; + tx.last_insert_rowid() + } + }; + files_ids::record(&tx, source_id, files_ids::CONTACT_CARD, &card.uid, id)?; + seen.insert(card.uid, id); + } + tx.commit()?; + listed_and_found(logger, &book.file, "cards", book.cards, chunks.len()); + } + let tx = conn.transaction()?; + collections.tally(&tx, "address_books", BOOK_COLUMNS, &mut out.collection)?; + items.tally(&tx, "contact_cards", CARD_COLUMNS, &mut out.item)?; + tx.commit()?; + Ok(out) +} + +pub fn import_calendars( + conn: &mut Connection, + root: &Path, + source_id: i64, + calendars: &[Collection], + logger: Logger, +) -> Result { + let mut out = Outcome { + collection: TypeCounts::default(), + item: TypeCounts::default(), + }; + let mut collections = Rerun::default(); + let mut items = Rerun::default(); + let mut seen: HashMap = HashMap::new(); + for (order, calendar) in calendars.iter().enumerate() { + let tx = conn.transaction()?; + let calendar_id = ensure_collection( + &tx, + source_id, + Kind::Calendars, + calendar, + order, + &mut collections, + &mut out.collection, + logger, + )?; + let Some(text) = read_part_file(root, &calendar.file, "calendars", logger) else { + out.item.failed += 1; + tx.commit()?; + continue; + }; + let entries = match calcard::ical_to_jscalendar_entries(&text) { + Ok(entries) => entries, + // A calendar with no events is written as an empty VCALENDAR. + Err(calcard::CalcardError::NoEntries) => Vec::new(), + Err(e) => { + logger.warn(&format!("{}: {e}", calendar.file)); + out.item.failed += 1; + tx.commit()?; + continue; + } + }; + let found = entries.len(); + for (ordinal, mut entry) in entries.into_iter().enumerate() { + let (is_draft, use_default_alerts, uid) = + calcard::strip_extracted_fields_from_event(&mut entry.data); + let uid = uid + .map(|u| u.trim().to_owned()) + .filter(|u| !u.is_empty()) + .unwrap_or_else(|| { + calcard::synthesise_uid(&format!("{}#{ordinal}", calendar.file)) + }); + // An occurrence sent on its own shares its series' uid. + let key = match entry.data.get("recurrenceId").and_then(Value::as_str) { + Some(recurrence) => serde_json::to_string(&[&uid, recurrence]).unwrap_or_default(), + None => uid, + }; + if let Some(&id) = seen.get(&key) { + add_to_collection(&tx, "calendar_events", "calendar_ids", id, calendar_id)?; + continue; + } + let ids = format!("[{calendar_id}]"); + let data = entry.data.to_string(); + let data_type = entry.data_type.as_column(); + let id = match files_ids::local_for(&tx, source_id, files_ids::CALENDAR_EVENT, &key)? { + Some(id) if row_exists(&tx, "calendar_events", id)? => { + items.before(&tx, "calendar_events", EVENT_COLUMNS, id)?; + tx.execute( + "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, + use_default_alerts = ?3, data = ?4, data_type = ?5 + WHERE id = ?6", + params![ + ids, + is_draft as i64, + use_default_alerts as i64, + data, + data_type, + id + ], + )?; + id + } + _ => { + tx.execute( + "INSERT INTO calendar_events + (calendar_ids, is_draft, use_default_alerts, data, data_type) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + ids, + is_draft as i64, + use_default_alerts as i64, + data, + data_type + ], + )?; + out.item.created += 1; + out.item.fetched += 1; + tx.last_insert_rowid() + } + }; + files_ids::record(&tx, source_id, files_ids::CALENDAR_EVENT, &key, id)?; + seen.insert(key, id); + } + tx.commit()?; + listed_and_found(logger, &calendar.file, "events", calendar.events, found); + } + let tx = conn.transaction()?; + collections.tally(&tx, "calendars", CALENDAR_COLUMNS, &mut out.collection)?; + items.tally(&tx, "calendar_events", EVENT_COLUMNS, &mut out.item)?; + tx.commit()?; + Ok(out) +} + +#[derive(Clone, Copy)] +enum Kind { + Contacts, + Calendars, +} + +impl Kind { + fn table(self) -> &'static str { + match self { + Kind::Contacts => "address_books", + Kind::Calendars => "calendars", + } + } + + fn type_name(self) -> &'static str { + match self { + Kind::Contacts => files_ids::ADDRESS_BOOK, + Kind::Calendars => files_ids::CALENDAR, + } + } + + fn object(self) -> ObjectType { + match self { + Kind::Contacts => ObjectType::AddressBook, + Kind::Calendars => ObjectType::Calendar, + } + } +} + +/// The address book or calendar for one manifest entry, keyed by its file. +#[allow(clippy::too_many_arguments)] +fn ensure_collection( + conn: &Connection, + source_id: i64, + kind: Kind, + entry: &Collection, + order: usize, + rerun: &mut Rerun, + counts: &mut TypeCounts, + logger: Logger, +) -> Result { + let table = kind.table(); + let existing = match files_ids::local_for(conn, source_id, kind.type_name(), &entry.file)? { + Some(id) if row_exists(conn, table, id)? => Some(id), + _ => None, + }; + let is_default = defaults::unique_default(conn, kind.object(), entry.default, existing)?; + if entry.default && !is_default { + logger.warn(&format!( + "{}: another {} is already the default, so this one is not", + entry.file, + match kind { + Kind::Contacts => "address book", + Kind::Calendars => "calendar", + } + )); + } + let blank = |v: &Option| v.clone().filter(|s| !s.is_empty()); + let id = match (kind, existing) { + (Kind::Contacts, Some(id)) => { + rerun.before(conn, table, BOOK_COLUMNS, id)?; + conn.execute( + "UPDATE address_books SET name = ?1, description = ?2, sort_order = ?3, + is_default = ?4 + WHERE id = ?5", + params![ + entry.name, + blank(&entry.description), + order as i64, + is_default as i64, + id + ], + )?; + id + } + (Kind::Contacts, None) => { + conn.execute( + "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed) + VALUES (?1, ?2, ?3, ?4, 1)", + params![ + entry.name, + blank(&entry.description), + order as i64, + is_default as i64 + ], + )?; + counts.created += 1; + counts.fetched += 1; + conn.last_insert_rowid() + } + (Kind::Calendars, Some(id)) => { + rerun.before(conn, table, CALENDAR_COLUMNS, id)?; + conn.execute( + "UPDATE calendars SET name = ?1, description = ?2, color = ?3, time_zone = ?4, + sort_order = ?5, is_default = ?6 + WHERE id = ?7", + params![ + entry.name, + blank(&entry.description), + blank(&entry.color), + blank(&entry.time_zone), + order as i64, + is_default as i64, + id + ], + )?; + id + } + (Kind::Calendars, None) => { + conn.execute( + "INSERT INTO calendars + (name, description, color, time_zone, sort_order, is_subscribed, + is_visible, is_default) + VALUES (?1, ?2, ?3, ?4, ?5, 1, 1, ?6)", + params![ + entry.name, + blank(&entry.description), + blank(&entry.color), + blank(&entry.time_zone), + order as i64, + is_default as i64 + ], + )?; + counts.created += 1; + counts.fetched += 1; + conn.last_insert_rowid() + } + }; + files_ids::record(conn, source_id, kind.type_name(), &entry.file, id)?; + Ok(id) +} + +fn add_to_collection( + conn: &Connection, + table: &str, + column: &str, + id: i64, + collection: i64, +) -> Result<(), Error> { + let ids: String = conn.query_row( + &format!("SELECT {column} FROM {table} WHERE id = ?1"), + params![id], + |r| r.get(0), + )?; + conn.execute( + &format!("UPDATE {table} SET {column} = ?1 WHERE id = ?2"), + params![cards::with_book(&ids, collection), id], + )?; + Ok(()) +} + +/// A text file the manifest names, if it is safely inside `part`. +pub(super) fn read_part_file(root: &Path, rel: &str, part: &str, logger: Logger) -> Option { + let path = match paths::resolve(root, rel, part) { + Ok(path) if paths::is_real_file(&path) => path, + Ok(_) => { + logger.warn(&format!("{rel}: not a file")); + return None; + } + Err(reason) => { + logger.warn(&reason); + return None; + } + }; + match fs::read_to_string(&path) { + Ok(text) => Some(text), + Err(e) => { + logger.warn(&format!("{rel}: {e}")); + None + } + } +} + +fn listed_and_found(logger: Logger, file: &str, what: &str, listed: Option, found: usize) { + if let Some(listed) = listed + && listed != found as u64 + { + logger.warn(&format!( + "{file}: the manifest lists {listed} {what} and {found} were found" + )); + } +} diff --git a/src/sync/import_files/mail.rs b/src/sync/import_files/mail.rs new file mode 100644 index 0000000..3ed25e4 --- /dev/null +++ b/src/sync/import_files/mail.rs @@ -0,0 +1,453 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! The mail part: a Maildir++ tree, read with the manifest's real folder +//! names, hierarchy, roles and subscriptions in place of what the directory +//! names could hold. A message kept in several folders is written to each; +//! copies with the same content become one message in several mailboxes, +//! as the archive holds them. File names are only read for their Maildir +//! flags: the specification says nothing else in them can be relied on. + +use std::collections::{BTreeSet, HashMap}; +use std::fs; +use std::path::Path; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::db::{blobs, cards, files_ids, roles}; +use crate::error::Error; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::TypeCounts; +use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::import_maildir::keywords::{ + custom_keywords_from_filename, flags_from_filename, read_keyword_names, translate_flags, + with_custom, +}; +use crate::sync::keys::index_to_json; + +use super::manifest::{Folder, Mail}; +use super::{Rerun, mtime_rfc3339, now_rfc3339, paths}; + +pub struct Outcome { + pub mailbox: TypeCounts, + pub email: TypeCounts, +} + +pub fn import( + conn: &mut Connection, + root: &Path, + source_id: i64, + mail: &Mail, + logger: Logger, +) -> Result { + let mut out = Outcome { + mailbox: TypeCounts::default(), + email: TypeCounts::default(), + }; + if paths::segments(&mail.path).map(|s| s.len()) != Ok(1) { + return Err(Error::Usage(format!( + "manifest.json: mail path {:?} is not a directory at the export's top", + mail.path + ))); + } + + // Parents before children, so each folder finds its parent made. + let mut folders: Vec<&Folder> = mail.folders.iter().collect(); + folders.sort_by_key(|f| real_path(f).len()); + let mut planned: Vec<(&Folder, i64)> = Vec::new(); + { + let tx = conn.transaction()?; + let mut tree = Tree { + conn: &tx, + source_id, + made: HashMap::new(), + rerun: Rerun::default(), + logger, + }; + for folder in folders { + match tree.ensure(&real_path(folder), Some(folder), &mut out.mailbox) { + Ok(id) => planned.push((folder, id)), + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!("folder {:?}: {e}", folder.name)); + out.mailbox.failed += 1; + } + } + } + tree.rerun + .tally(&tx, "mailboxes", MAILBOX_COLUMNS, &mut out.mailbox)?; + tx.commit()?; + } + + let mut messages = Messages { + source_id, + seen: HashMap::new(), + rerun: Rerun::default(), + copies: 0, + logger, + }; + for (folder, mailbox) in &planned { + let dir = match paths::resolve(root, &folder.dir, &mail.path) { + Ok(dir) if paths::is_real_dir(&dir) => dir, + Ok(_) => { + logger.warn(&format!( + "folder {:?}: {} is not a directory", + folder.name, folder.dir + )); + out.email.failed += 1; + continue; + } + Err(reason) => { + logger.warn(&format!("folder {:?}: {reason}", folder.name)); + out.email.failed += 1; + continue; + } + }; + let tx = conn.transaction()?; + let found = match messages.folder(&tx, &dir, *mailbox, &folder.name, &mut out.email) { + Ok(found) => found, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!("folder {:?}: {e}", folder.name)); + out.email.failed += 1; + 0 + } + }; + tx.commit()?; + if let Some(listed) = folder.messages + && listed != found + { + logger.warn(&format!( + "folder {:?}: the manifest lists {listed} messages and {found} were found", + folder.name + )); + } + if let Some(dropped) = folder.keywords_dropped.filter(|n| *n > 0) { + logger.warn(&format!( + "folder {:?}: the export had no room for {dropped} keywords on its messages \ + (Maildir holds 26 per folder), so those are not on them", + folder.name + )); + } + } + { + let tx = conn.transaction()?; + messages + .rerun + .tally(&tx, "emails", EMAIL_COLUMNS, &mut out.email)?; + tx.commit()?; + } + if messages.copies > 0 && logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "mail: {} copies of messages kept in more than one folder were read as one \ + message in each of those folders", + messages.copies + ); + } + Ok(out) +} + +const MAILBOX_COLUMNS: &str = "name, parent_id, role, is_subscribed"; +const EMAIL_COLUMNS: &str = "mailbox_ids, keywords"; + +/// The folder's real names from the top down; the manifest may leave the +/// path out of a top-level folder. +fn real_path(folder: &Folder) -> Vec { + if folder.path.is_empty() { + vec![folder.name.clone()] + } else { + folder.path.clone() + } +} + +struct Tree<'a> { + conn: &'a Connection, + source_id: i64, + made: HashMap, i64>, + rerun: Rerun, + logger: Logger, +} + +impl Tree<'_> { + /// The mailbox for `path`, made or brought up to date. A parent the + /// manifest doesn't list is made too, so its children have one. + fn ensure( + &mut self, + path: &[String], + folder: Option<&Folder>, + counts: &mut TypeCounts, + ) -> Result { + if folder.is_none() + && let Some(id) = self.made.get(path) + { + return Ok(*id); + } + let Some((name, parents)) = path.split_last() else { + return Err(Error::Partial("a folder with no name".to_owned())); + }; + let parent = if parents.is_empty() { + None + } else { + Some(self.ensure(parents, None, counts)?) + }; + let key = serde_json::to_string(path).unwrap_or_default(); + let existing = + match files_ids::local_for(self.conn, self.source_id, files_ids::MAILBOX, &key)? { + Some(id) if row_exists(self.conn, "mailboxes", id)? => Some(id), + _ => self + .conn + .query_row( + "SELECT id FROM mailboxes WHERE parent_id IS ?1 AND name = ?2", + params![parent, name], + |r| r.get::<_, i64>(0), + ) + .optional()?, + }; + let Some(folder) = folder else { + // A parent only: one the manifest doesn't list. + if let Some(id) = existing { + self.made.insert(path.to_vec(), id); + files_ids::record(self.conn, self.source_id, files_ids::MAILBOX, &key, id)?; + return Ok(id); + } + if self.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "folder {:?}: not in the manifest; made so its folders have a parent", + path.join("/") + ); + } + self.conn.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, NULL, 0, 1)", + params![name, parent], + )?; + let id = self.conn.last_insert_rowid(); + files_ids::record(self.conn, self.source_id, files_ids::MAILBOX, &key, id)?; + self.made.insert(path.to_vec(), id); + counts.created += 1; + counts.fetched += 1; + return Ok(id); + }; + + let wanted_role = folder + .role + .as_deref() + .map(str::trim) + .filter(|r| !r.is_empty()) + .map(str::to_ascii_lowercase); + let role = roles::unique_role(self.conn, wanted_role.as_deref(), existing)?; + if let (Some(wanted), None) = (&wanted_role, &role) { + self.logger.warn(&format!( + "folder {:?}: another folder already has the role {wanted:?}, so this one \ + is left without it", + path.join("/") + )); + } + let subscribed = folder.subscribed as i64; + let id = match existing { + Some(id) => { + self.rerun + .before(self.conn, "mailboxes", MAILBOX_COLUMNS, id)?; + self.conn.execute( + "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4 + WHERE id = ?5", + params![name, parent, role, subscribed, id], + )?; + id + } + None => { + self.conn.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, 0, ?4)", + params![name, parent, role, subscribed], + )?; + counts.created += 1; + counts.fetched += 1; + self.conn.last_insert_rowid() + } + }; + files_ids::record(self.conn, self.source_id, files_ids::MAILBOX, &key, id)?; + self.made.insert(path.to_vec(), id); + Ok(id) + } +} + +struct Messages { + source_id: i64, + /// Each message read this run, by content hash. + seen: HashMap, + rerun: Rerun, + copies: u64, + logger: Logger, +} + +impl Messages { + /// Every message in one folder's `cur/` and `new/`; returns how many + /// were found. + fn folder( + &mut self, + conn: &Connection, + dir: &Path, + mailbox: i64, + folder_name: &str, + counts: &mut TypeCounts, + ) -> Result { + let keyword_file = dir.join("dovecot-keywords"); + let names = if paths::is_real_file(&keyword_file) { + read_keyword_names(dir) + } else { + if keyword_file.symlink_metadata().is_ok() { + self.logger.warn(&format!( + "folder {folder_name:?}: its dovecot-keywords is not a plain file and \ + is not read" + )); + } + vec![None; 26] + }; + let mut found = 0; + for sub in ["cur", "new"] { + let sub_dir = dir.join(sub); + if !paths::is_real_dir(&sub_dir) { + if sub_dir.symlink_metadata().is_ok() { + self.logger.warn(&format!( + "folder {folder_name:?}: {sub}/ is not a directory and is not read" + )); + counts.failed += 1; + } + continue; + } + let mut entries: Vec = fs::read_dir(&sub_dir) + .and_then(|d| d.collect::>()) + .map_err(|e| Error::Partial(format!("reading {sub}/: {e}")))?; + entries.sort_by_key(fs::DirEntry::file_name); + for entry in entries { + let file_name = entry.file_name().to_string_lossy().into_owned(); + if file_name.starts_with('.') { + continue; + } + let file_type = entry + .file_type() + .map_err(|e| Error::Partial(format!("{sub}/{file_name}: {e}")))?; + if file_type.is_symlink() { + self.logger.warn(&format!( + "folder {folder_name:?}: {sub}/{file_name} is a symbolic link and is \ + not followed" + )); + counts.skipped += 1; + continue; + } + if !file_type.is_file() { + continue; + } + found += 1; + if let Err(e) = self.message(conn, &entry, &file_name, &names, mailbox, counts) { + if e.aborts_run() { + return Err(e); + } + self.logger + .warn(&format!("folder {folder_name:?}: {sub}/{file_name}: {e}")); + counts.failed += 1; + } + } + } + Ok(found) + } + + fn message( + &mut self, + conn: &Connection, + entry: &fs::DirEntry, + file_name: &str, + keyword_names: &[Option], + mailbox: i64, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let bytes = fs::read(entry.path()).map_err(|e| Error::Partial(e.to_string()))?; + let hash = blake3::hash(&bytes).to_hex().to_string(); + let keywords: BTreeSet = with_custom( + translate_flags(&flags_from_filename(file_name), true), + &custom_keywords_from_filename(file_name, keyword_names), + ) + .keywords + .into_iter() + .collect(); + + if let Some(&id) = self.seen.get(&hash) { + // Another copy of a message already read this run. + let (mailbox_ids, current): (String, String) = conn.query_row( + "SELECT mailbox_ids, keywords FROM emails WHERE id = ?1", + params![id], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + let mut merged: BTreeSet = serde_json::from_str::>(¤t) + .unwrap_or_default() + .into_iter() + .collect(); + merged.extend(keywords); + conn.execute( + "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 WHERE id = ?3", + params![ + cards::with_book(&mailbox_ids, mailbox), + keywords_json(&merged), + id + ], + )?; + self.copies += 1; + return Ok(()); + } + + let mailbox_ids = format!("[{mailbox}]"); + if let Some(id) = files_ids::local_for(conn, self.source_id, files_ids::EMAIL, &hash)? + && row_exists(conn, "emails", id)? + { + // Read on an earlier run: now in the folders it is in now. + self.rerun.before(conn, "emails", EMAIL_COLUMNS, id)?; + conn.execute( + "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 WHERE id = ?3", + params![mailbox_ids, keywords_json(&keywords), id], + )?; + self.seen.insert(hash, id); + return Ok(()); + } + + let (index, date_header) = email_meta_from_blob(&bytes); + // The file's modification time is when the message arrived. + let received_at = entry + .metadata() + .ok() + .and_then(|m| mtime_rfc3339(&m)) + .or(date_header) + .unwrap_or_else(now_rfc3339); + let blob = blobs::intern_blob(conn, &bytes)?; + conn.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + blob, + received_at, + mailbox_ids, + keywords_json(&keywords), + index_to_json(&index) + ], + )?; + let id = conn.last_insert_rowid(); + files_ids::record(conn, self.source_id, files_ids::EMAIL, &hash, id)?; + self.seen.insert(hash, id); + counts.created += 1; + counts.fetched += 1; + Ok(()) + } +} + +fn keywords_json(keywords: &BTreeSet) -> String { + serde_json::to_string(keywords).unwrap_or_else(|_| "[]".to_owned()) +} + +pub(super) fn row_exists(conn: &Connection, table: &str, id: i64) -> Result { + Ok(conn + .prepare(&format!("SELECT 1 FROM {table} WHERE id = ?1"))? + .exists(params![id])?) +} diff --git a/src/sync/import_files/manifest.rs b/src/sync/import_files/manifest.rs new file mode 100644 index 0000000..2c9d01e --- /dev/null +++ b/src/sync/import_files/manifest.rs @@ -0,0 +1,317 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `manifest.json`, as +//! defines it. A part that is missing wasn't exported; a part that is there +//! and empty says the account had none. Members this release doesn't know +//! are ignored, so a later writer can add them. + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +use serde::Deserialize; +use serde_json::Value; + +use crate::error::Error; +use crate::export_files::{FORMAT, FORMAT_VERSION}; + +/// What inbuxa-migrate 27.1.0-rc.2 wrote as the format before it was named +/// by the specification's address. The specification lets readers accept it. +pub const LEGACY_FORMAT: &str = "inbuxa-migrate-files"; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + #[serde(default)] + pub created_by: Option, + #[serde(default)] + pub created: Option, + #[serde(default)] + pub sources: Vec, + #[serde(default)] + pub mail: Option, + #[serde(default)] + pub contacts: Option>, + #[serde(default)] + pub calendars: Option>, + #[serde(default)] + pub sieve: Option>, + #[serde(default)] + pub files: Option, + #[serde(default)] + pub identities: Option>, + #[serde(default)] + pub participant_identities: Option>, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Source { + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub url: Option, + #[serde(default)] + pub account_name: Option, + #[serde(default)] + pub username: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Mail { + pub layout: String, + pub path: String, + #[serde(default)] + pub folders: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Folder { + pub name: String, + #[serde(default)] + pub path: Vec, + pub dir: String, + #[serde(default)] + pub role: Option, + #[serde(default = "yes")] + pub subscribed: bool, + #[serde(default)] + pub messages: Option, + #[serde(default)] + pub keywords_dropped: Option, +} + +/// An address book or a calendar: the members both have, and those only a +/// calendar has. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Collection { + pub name: String, + #[serde(default)] + pub description: Option, + pub file: String, + #[serde(default)] + pub default: bool, + #[serde(default)] + pub color: Option, + #[serde(default)] + pub time_zone: Option, + #[serde(default)] + pub cards: Option, + #[serde(default)] + pub events: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Script { + pub name: String, + pub file: String, + #[serde(default)] + pub active: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Files { + pub path: String, + #[serde(default)] + pub media_types: BTreeMap, + #[serde(default)] + pub links: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Link { + pub path: String, + #[serde(default)] + pub target: Value, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Identity { + #[serde(default)] + pub name: String, + pub email: String, + #[serde(default)] + pub reply_to: Value, + #[serde(default)] + pub bcc: Value, + #[serde(default)] + pub text_signature: String, + #[serde(default)] + pub html_signature: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ParticipantIdentity { + #[serde(default)] + pub name: String, + pub calendar_address: String, + #[serde(default)] + pub default: bool, +} + +fn yes() -> bool { + true +} + +/// The manifest of the export at `root`. A tree without one is unfinished +/// and is refused, and so is a format or version this release doesn't read. +pub fn read(root: &Path) -> Result { + let path = root.join("manifest.json"); + match fs::symlink_metadata(&path) { + Ok(meta) if meta.file_type().is_symlink() => { + return Err(Error::Usage(format!( + "{}: manifest.json is a symbolic link; an export's files are never links", + root.display() + ))); + } + Ok(meta) if !meta.is_file() => { + return Err(Error::Usage(format!( + "{}: manifest.json is not a file", + root.display() + ))); + } + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::Usage(format!( + "{} has no manifest.json: either the export was interrupted (a writer \ + adds the manifest last) or this is not an account export", + root.display() + ))); + } + Err(e) => return Err(Error::Io(e)), + } + let text = fs::read_to_string(&path)?; + parse(&text) +} + +pub fn parse(text: &str) -> Result { + let value: Value = serde_json::from_str(text) + .map_err(|e| Error::Usage(format!("manifest.json is not valid JSON: {e}")))?; + let Value::Object(map) = &value else { + return Err(Error::Usage( + "manifest.json is not a JSON object".to_owned(), + )); + }; + match map.get("format").and_then(Value::as_str) { + Some(f) if f == FORMAT || f == LEGACY_FORMAT => {} + Some(other) => { + return Err(Error::Usage(format!( + "manifest.json names the format {other:?}, not the account export layout \ + ({FORMAT})" + ))); + } + None => { + return Err(Error::Usage( + "manifest.json has no format; this is not an account export".to_owned(), + )); + } + } + match map.get("formatVersion").and_then(Value::as_u64) { + Some(v) if v == u64::from(FORMAT_VERSION) => {} + Some(v) if v > u64::from(FORMAT_VERSION) => { + return Err(Error::Usage(format!( + "this export follows version {v} of the layout, and this release reads \ + version {FORMAT_VERSION}; use a newer inbuxa-migrate" + ))); + } + _ => { + return Err(Error::Usage( + "manifest.json has no usable formatVersion".to_owned(), + )); + } + } + let manifest: Manifest = + serde_json::from_value(value).map_err(|e| Error::Usage(format!("manifest.json: {e}")))?; + if let Some(mail) = &manifest.mail + && !mail.layout.eq_ignore_ascii_case("maildir++") + { + return Err(Error::Usage(format!( + "manifest.json: mail layout {:?} is not one this release reads (maildir++)", + mail.layout + ))); + } + Ok(manifest) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn err(text: &str) -> String { + parse(text).unwrap_err().to_string() + } + + #[test] + fn the_format_is_the_spec_address_or_the_old_name() { + let m = parse(&format!(r#"{{"format":"{FORMAT}","formatVersion":1}}"#)).unwrap(); + assert!(m.mail.is_none() && m.contacts.is_none(), "nothing exported"); + parse(r#"{"format":"inbuxa-migrate-files","formatVersion":1}"#).unwrap(); + assert!( + err(r#"{"format":"something-else","formatVersion":1}"#).contains("not the account") + ); + assert!(err(r#"{"formatVersion":1}"#).contains("no format")); + } + + #[test] + fn a_later_version_is_refused_rather_than_guessed_at() { + let e = err(&format!(r#"{{"format":"{FORMAT}","formatVersion":2}}"#)); + assert!(e.contains("version 2") && e.contains("newer"), "{e}"); + assert!( + err(&format!(r#"{{"format":"{FORMAT}","formatVersion":"1"}}"#)) + .contains("formatVersion") + ); + assert!(err(&format!(r#"{{"format":"{FORMAT}"}}"#)).contains("formatVersion")); + } + + #[test] + fn unknown_members_are_ignored_and_empty_parts_kept_apart_from_missing_ones() { + let m = parse(&format!( + r#"{{"format":"{FORMAT}","formatVersion":1,"futureThing":{{"x":1}}, + "contacts":[],"mail":{{"layout":"maildir++","path":"mail","folders":[ + {{"name":"Inbox","path":["Inbox"],"dir":"mail","role":"inbox", + "subscribed":true,"messages":3,"newMember":true}}]}}}}"# + )) + .unwrap(); + assert_eq!(m.contacts.as_deref().map(<[Collection]>::len), Some(0)); + assert!(m.calendars.is_none()); + let mail = m.mail.unwrap(); + assert_eq!(mail.folders[0].role.as_deref(), Some("inbox")); + } + + #[test] + fn another_mail_layout_is_refused() { + let e = err(&format!( + r#"{{"format":"{FORMAT}","formatVersion":1,"mail":{{"layout":"mbox","path":"mail"}}}}"# + )); + assert!(e.contains("mbox"), "{e}"); + } + + #[test] + fn a_missing_or_linked_manifest_is_refused() { + let td = tempfile::tempdir().unwrap(); + let e = read(td.path()).unwrap_err().to_string(); + assert!(e.contains("interrupted"), "{e}"); + #[cfg(unix)] + { + let real = td.path().join("real.json"); + fs::write( + &real, + format!(r#"{{"format":"{FORMAT}","formatVersion":1}}"#), + ) + .unwrap(); + std::os::unix::fs::symlink(&real, td.path().join("manifest.json")).unwrap(); + let e = read(td.path()).unwrap_err().to_string(); + assert!(e.contains("symbolic link"), "{e}"); + } + } +} diff --git a/src/sync/import_files/other.rs b/src/sync/import_files/other.rs new file mode 100644 index 0000000..c3d1214 --- /dev/null +++ b/src/sync/import_files/other.rs @@ -0,0 +1,532 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Sieve scripts with the one that was active, the file tree with its +//! media types and links, and the identities, which only the manifest +//! holds. + +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use rusqlite::{Connection, OptionalExtension, params}; +use serde_json::Value; + +use crate::db::{blobs, defaults, files_ids}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::keys::{identity_key, participant_identity_key}; +use crate::types::ObjectType; + +use super::mail::row_exists; +use super::manifest::{Files, Identity, ParticipantIdentity, Script}; +use super::{Rerun, mtime_rfc3339, paths}; + +const SCRIPT_COLUMNS: &str = "blob_id, is_active"; +const NODE_COLUMNS: &str = "parent_id, node_type, blob_id, target, name, media_type, modified"; +const IDENTITY_COLUMNS: &str = "name, email, reply_to, bcc, text_signature, html_signature"; +const PARTICIPANT_COLUMNS: &str = "name, calendar_address, is_default"; + +pub fn import_sieve( + conn: &mut Connection, + root: &Path, + source_id: i64, + scripts: &[Script], + logger: Logger, +) -> Result { + let mut counts = TypeCounts::default(); + let mut rerun = Rerun::default(); + let tx = conn.transaction()?; + let mut active: Option<&str> = None; + for script in scripts { + let path = match paths::resolve(root, &script.file, "sieve") { + Ok(path) if paths::is_real_file(&path) => path, + Ok(_) => { + logger.warn(&format!("{}: not a file", script.file)); + counts.failed += 1; + continue; + } + Err(reason) => { + logger.warn(&format!("Sieve script {:?}: {reason}", script.name)); + counts.failed += 1; + continue; + } + }; + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(e) => { + logger.warn(&format!("{}: {e}", script.file)); + counts.failed += 1; + continue; + } + }; + let is_active = script.active && active.is_none(); + if script.active && !is_active { + logger.warn(&format!( + "Sieve script {:?}: the manifest marks more than one script active; only \ + {:?} is", + script.name, + active.unwrap_or_default() + )); + } + if is_active { + active = Some(&script.name); + } + let blob = blobs::intern_blob(&tx, &bytes)?; + let existing = + match files_ids::local_for(&tx, source_id, files_ids::SIEVE_SCRIPT, &script.name)? { + Some(id) if row_exists(&tx, "sieve_scripts", id)? => Some(id), + // Names are unique in the archive. + _ => tx + .query_row( + "SELECT id FROM sieve_scripts WHERE name = ?1", + params![script.name], + |r| r.get::<_, i64>(0), + ) + .optional()?, + }; + if let Some(id) = existing { + rerun.before(&tx, "sieve_scripts", SCRIPT_COLUMNS, id)?; + } + if is_active { + // One script runs at a time. + tx.execute( + "UPDATE sieve_scripts SET is_active = 0 WHERE is_active = 1 AND id IS NOT ?1", + params![existing], + )?; + } + let id = match existing { + Some(id) => { + tx.execute( + "UPDATE sieve_scripts SET blob_id = ?1, is_active = ?2 WHERE id = ?3", + params![blob, is_active as i64, id], + )?; + id + } + None => { + tx.execute( + "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)", + params![script.name, is_active as i64, blob], + )?; + counts.created += 1; + counts.fetched += 1; + tx.last_insert_rowid() + } + }; + files_ids::record(&tx, source_id, files_ids::SIEVE_SCRIPT, &script.name, id)?; + } + rerun.tally(&tx, "sieve_scripts", SCRIPT_COLUMNS, &mut counts)?; + tx.commit()?; + Ok(counts) +} + +/// The file tree, walked without following links, then the links the +/// manifest lists, made as archive nodes and never on disk. +pub fn import_files( + conn: &mut Connection, + root: &Path, + source_id: i64, + files: &Files, + created: &str, + logger: Logger, +) -> Result { + let mut counts = TypeCounts::default(); + let part = match paths::segments(&files.path).as_deref() { + Ok([part]) => (*part).to_owned(), + _ => { + return Err(Error::Usage(format!( + "manifest.json: files path {:?} is not a directory at the export's top", + files.path + ))); + } + }; + let tx = conn.transaction()?; + let mut tree = FileTree { + conn: &tx, + source_id, + files, + dirs: HashMap::new(), + rerun: Rerun::default(), + created, + logger, + }; + // A writer with no files to write may not make the directory at all. + let base = root.join(&part); + match fs::symlink_metadata(&base) { + Ok(meta) if meta.is_dir() => tree.walk(&base, &part, None, &mut counts)?, + Ok(_) => { + logger.warn(&format!("{part}/ is not a directory and is not read")); + counts.failed += 1; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(Error::Io(e)), + } + for link in &files.links { + if let Err(reason) = tree.link(&part, &link.path, &link.target, &mut counts) { + logger.warn(&format!("link {:?}: {reason}", link.path)); + counts.failed += 1; + } + } + let rerun = std::mem::take(&mut tree.rerun); + rerun.tally(&tx, "file_nodes", NODE_COLUMNS, &mut counts)?; + tx.commit()?; + Ok(counts) +} + +struct FileTree<'a> { + conn: &'a Connection, + source_id: i64, + files: &'a Files, + /// Each directory made, by its path in the export. + dirs: HashMap, + rerun: Rerun, + /// When the export was written: the time for what has none of its own. + created: &'a str, + logger: Logger, +} + +struct Node<'a> { + parent: Option, + node_type: &'a str, + blob: Option, + target: Option, + name: &'a str, + media_type: Option<&'a str>, + created: String, + modified: Option, +} + +impl FileTree<'_> { + fn walk( + &mut self, + dir: &Path, + rel: &str, + parent: Option, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let mut entries: Vec = + match fs::read_dir(dir).and_then(|d| d.collect::>()) { + Ok(entries) => entries, + Err(e) => { + self.logger.warn(&format!("{rel}/: {e}")); + counts.failed += 1; + return Ok(()); + } + }; + entries.sort_by_key(fs::DirEntry::file_name); + for entry in entries { + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + self.logger.warn(&format!( + "{rel}/{}: the name is not UTF-8 and is skipped", + entry.file_name().to_string_lossy() + )); + counts.failed += 1; + continue; + }; + let rel_path = format!("{rel}/{name}"); + let Ok(file_type) = entry.file_type() else { + self.logger + .warn(&format!("{rel_path}: its type can't be read")); + counts.failed += 1; + continue; + }; + if file_type.is_symlink() { + self.logger.warn(&format!( + "{rel_path} is a symbolic link and is not followed; an export's links \ + are listed in its manifest" + )); + counts.skipped += 1; + continue; + } + if file_type.is_dir() { + // A writer sets the times of files, not of directories, so a + // directory's time is only when the export was written. + let id = self.node( + &rel_path, + Node { + parent, + node_type: "directory", + blob: None, + target: None, + name: &name, + media_type: None, + created: self.created.to_owned(), + modified: None, + }, + counts, + )?; + self.dirs.insert(rel_path.clone(), id); + self.walk(&entry.path(), &rel_path, Some(id), counts)?; + } else if file_type.is_file() { + let modified = entry.metadata().ok().and_then(|m| mtime_rfc3339(&m)); + let created = modified.clone().unwrap_or_else(|| self.created.to_owned()); + let bytes = match fs::read(entry.path()) { + Ok(bytes) => bytes, + Err(e) => { + self.logger.warn(&format!("{rel_path}: {e}")); + counts.failed += 1; + continue; + } + }; + let blob = blobs::intern_blob(self.conn, &bytes)?; + let media_type = self.files.media_types.get(&rel_path).map(String::as_str); + self.node( + &rel_path, + Node { + parent, + node_type: "file", + blob: Some(blob), + target: None, + name: &name, + media_type, + created, + modified, + }, + counts, + )?; + } else { + self.logger + .warn(&format!("{rel_path} is not a file or directory; skipped")); + counts.skipped += 1; + } + } + Ok(()) + } + + fn link( + &mut self, + part: &str, + rel: &str, + target: &Value, + counts: &mut TypeCounts, + ) -> Result<(), String> { + let segments = paths::segments(rel)?; + let [first, parents @ .., name] = segments.as_slice() else { + return Err("names no file".to_owned()); + }; + if *first != part { + return Err(format!("is outside {part}/")); + } + let parent = if parents.is_empty() { + None + } else { + let parent_rel = format!("{part}/{}", parents.join("/")); + match self.dirs.get(&parent_rel) { + Some(id) => Some(*id), + None => return Err(format!("its directory {parent_rel} is not in the export")), + } + }; + let target = (!target.is_null()).then(|| target.to_string()); + self.node( + rel, + Node { + parent, + node_type: "symlink", + blob: None, + target, + name, + media_type: None, + created: self.created.to_owned(), + modified: None, + }, + counts, + ) + .map(|_| ()) + .map_err(|e| e.to_string()) + } + + fn node(&mut self, key: &str, node: Node<'_>, counts: &mut TypeCounts) -> Result { + let existing = + match files_ids::local_for(self.conn, self.source_id, files_ids::FILE_NODE, key)? { + Some(id) if row_exists(self.conn, "file_nodes", id)? => Some(id), + // Names are unique within a directory. + _ => self + .conn + .query_row( + "SELECT id FROM file_nodes WHERE parent_id IS ?1 AND name = ?2", + params![node.parent, node.name], + |r| r.get::<_, i64>(0), + ) + .optional()?, + }; + let id = match existing { + Some(id) => { + self.rerun + .before(self.conn, "file_nodes", NODE_COLUMNS, id)?; + self.conn.execute( + "UPDATE file_nodes SET parent_id = ?1, node_type = ?2, blob_id = ?3, + target = ?4, name = ?5, media_type = ?6, modified = ?7 + WHERE id = ?8", + params![ + node.parent, + node.node_type, + node.blob, + node.target, + node.name, + node.media_type, + node.modified, + id + ], + )?; + id + } + None => { + self.conn.execute( + "INSERT INTO file_nodes + (parent_id, node_type, blob_id, target, name, media_type, created, modified) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + node.parent, + node.node_type, + node.blob, + node.target, + node.name, + node.media_type, + node.created, + node.modified + ], + )?; + counts.created += 1; + counts.fetched += 1; + self.conn.last_insert_rowid() + } + }; + files_ids::record(self.conn, self.source_id, files_ids::FILE_NODE, key, id)?; + Ok(id) + } +} + +pub fn import_identities( + conn: &mut Connection, + source_id: i64, + identities: &[Identity], +) -> Result { + let mut counts = TypeCounts::default(); + let mut rerun = Rerun::default(); + let tx = conn.transaction()?; + for identity in identities { + let key = hex(&identity_key(&identity.name, &identity.email)); + let addresses = |v: &Value| (!v.is_null()).then(|| v.to_string()); + let (reply_to, bcc) = (addresses(&identity.reply_to), addresses(&identity.bcc)); + let id = match files_ids::local_for(&tx, source_id, files_ids::IDENTITY, &key)? { + Some(id) if row_exists(&tx, "identities", id)? => { + rerun.before(&tx, "identities", IDENTITY_COLUMNS, id)?; + tx.execute( + "UPDATE identities SET name = ?1, email = ?2, reply_to = ?3, bcc = ?4, + text_signature = ?5, html_signature = ?6 + WHERE id = ?7", + params![ + identity.name, + identity.email, + reply_to, + bcc, + identity.text_signature, + identity.html_signature, + id + ], + )?; + id + } + _ => { + tx.execute( + "INSERT INTO identities + (name, email, reply_to, bcc, text_signature, html_signature) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + identity.name, + identity.email, + reply_to, + bcc, + identity.text_signature, + identity.html_signature + ], + )?; + counts.created += 1; + counts.fetched += 1; + tx.last_insert_rowid() + } + }; + files_ids::record(&tx, source_id, files_ids::IDENTITY, &key, id)?; + } + rerun.tally(&tx, "identities", IDENTITY_COLUMNS, &mut counts)?; + tx.commit()?; + Ok(counts) +} + +pub fn import_participant_identities( + conn: &mut Connection, + source_id: i64, + identities: &[ParticipantIdentity], + logger: Logger, +) -> Result { + let mut counts = TypeCounts::default(); + let mut rerun = Rerun::default(); + let tx = conn.transaction()?; + for identity in identities { + let key = hex(&participant_identity_key( + &identity.calendar_address, + &identity.name, + )); + let existing = + match files_ids::local_for(&tx, source_id, files_ids::PARTICIPANT_IDENTITY, &key)? { + Some(id) if row_exists(&tx, "participant_identities", id)? => Some(id), + _ => None, + }; + let is_default = defaults::unique_default( + &tx, + ObjectType::ParticipantIdentity, + identity.default, + existing, + )?; + if identity.default && !is_default { + logger.warn(&format!( + "calendar identity {:?}: another is already the default, so this one is not", + identity.calendar_address + )); + } + let id = match existing { + Some(id) => { + rerun.before(&tx, "participant_identities", PARTICIPANT_COLUMNS, id)?; + tx.execute( + "UPDATE participant_identities SET name = ?1, calendar_address = ?2, + is_default = ?3 + WHERE id = ?4", + params![ + identity.name, + identity.calendar_address, + is_default as i64, + id + ], + )?; + id + } + None => { + tx.execute( + "INSERT INTO participant_identities (name, calendar_address, is_default) + VALUES (?1, ?2, ?3)", + params![identity.name, identity.calendar_address, is_default as i64], + )?; + counts.created += 1; + counts.fetched += 1; + tx.last_insert_rowid() + } + }; + files_ids::record(&tx, source_id, files_ids::PARTICIPANT_IDENTITY, &key, id)?; + } + rerun.tally( + &tx, + "participant_identities", + PARTICIPANT_COLUMNS, + &mut counts, + )?; + tx.commit()?; + Ok(counts) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/src/sync/import_files/paths.rs b/src/sync/import_files/paths.rs new file mode 100644 index 0000000..a93c72a --- /dev/null +++ b/src/sync/import_files/paths.rs @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Paths from the manifest are data: whoever wrote the export chose them. +//! Each is checked to stay inside its part of the tree, and no part of it +//! may be a symbolic link, before anything is opened. + +use std::fs; +use std::path::{Component, Path, PathBuf}; + +/// `rel`, a `/`-separated path from the manifest, under `root`. It must +/// start with `part` (`mail`, `contacts` and so on), climb nowhere, and +/// pass through no symbolic link. What it names must exist. +pub fn resolve(root: &Path, rel: &str, part: &str) -> Result { + let segments = segments(rel)?; + if segments.first() != Some(&part) { + return Err(format!("{rel:?} is outside {part}/")); + } + let mut path = root.to_path_buf(); + for segment in &segments { + path.push(segment); + match fs::symlink_metadata(&path) { + Ok(meta) if meta.file_type().is_symlink() => { + return Err(format!( + "{rel:?} passes through a symbolic link, which is never followed" + )); + } + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(format!("{rel:?} is not in the export")); + } + Err(e) => return Err(format!("{rel:?}: {e}")), + } + } + Ok(path) +} + +/// The segments of a manifest path, refused if any could climb out or +/// mean something other than a plain name. +pub fn segments(rel: &str) -> Result, String> { + if rel.is_empty() { + return Err("an empty path".to_owned()); + } + let mut out = Vec::new(); + for segment in rel.split('/') { + let plain = !segment.is_empty() + && segment != "." + && segment != ".." + && !segment.contains(['\\', '\0']) + && matches!( + Path::new(segment) + .components() + .collect::>() + .as_slice(), + [Component::Normal(_)] + ); + if !plain { + return Err(format!("{rel:?} is not a plain relative path")); + } + out.push(segment); + } + Ok(out) +} + +/// A directory, and not a link to one. +pub fn is_real_dir(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok_and(|m| m.is_dir()) +} + +/// A regular file, and not a link to one. +pub fn is_real_file(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok_and(|m| m.is_file()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paths_that_climb_or_jump_are_refused() { + for bad in [ + "", + "/etc/passwd", + "mail/../../etc", + "mail/./x", + "mail//x", + "mail/a\\..\\b", + "..", + ] { + assert!(segments(bad).is_err(), "{bad:?}"); + } + assert_eq!(segments("mail/.INBOX.x").unwrap(), ["mail", ".INBOX.x"]); + } + + #[test] + fn a_path_must_stay_in_its_part_and_exist() { + let td = tempfile::tempdir().unwrap(); + fs::create_dir_all(td.path().join("contacts")).unwrap(); + fs::write(td.path().join("contacts/a.vcf"), "").unwrap(); + fs::create_dir_all(td.path().join("calendars")).unwrap(); + fs::write(td.path().join("calendars/b.ics"), "").unwrap(); + assert!(resolve(td.path(), "contacts/a.vcf", "contacts").is_ok()); + let e = resolve(td.path(), "calendars/b.ics", "contacts").unwrap_err(); + assert!(e.contains("outside contacts/"), "{e}"); + let e = resolve(td.path(), "contacts/missing.vcf", "contacts").unwrap_err(); + assert!(e.contains("not in the export"), "{e}"); + } + + #[cfg(unix)] + #[test] + fn a_link_anywhere_on_the_way_is_refused() { + let td = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("secret"), "x").unwrap(); + fs::create_dir_all(td.path().join("sieve")).unwrap(); + std::os::unix::fs::symlink( + outside.path().join("secret"), + td.path().join("sieve/a.sieve"), + ) + .unwrap(); + std::os::unix::fs::symlink(outside.path(), td.path().join("contacts")).unwrap(); + let e = resolve(td.path(), "sieve/a.sieve", "sieve").unwrap_err(); + assert!(e.contains("symbolic link"), "{e}"); + let e = resolve(td.path(), "contacts/secret", "contacts").unwrap_err(); + assert!(e.contains("symbolic link"), "{e}"); + assert!(!is_real_file(&td.path().join("sieve/a.sieve"))); + assert!(!is_real_dir(&td.path().join("contacts"))); + } +} diff --git a/src/sync/import_takeout/contacts.rs b/src/sync/import_takeout/contacts.rs index 4d13149..231eb71 100644 --- a/src/sync/import_takeout/contacts.rs +++ b/src/sync/import_takeout/contacts.rs @@ -160,7 +160,8 @@ fn insert_card( Ok(()) } -fn split_vcards(text: &str) -> Vec { +/// The vCards in one file, each as its own text. +pub fn split_vcards(text: &str) -> Vec { let mut chunks: Vec = Vec::new(); let mut current: Option = None; for raw_line in text.lines() { diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 90d93a1..264cf1d 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -14,6 +14,7 @@ pub mod export_managesieve; pub mod import_dav; pub mod import_exchange_ews; pub mod import_exchange_graph; +pub mod import_files; pub mod import_imap; pub mod import_jmap; pub mod import_maildir; diff --git a/tests/import_files.rs b/tests/import_files.rs new file mode 100644 index 0000000..00fa61c --- /dev/null +++ b/tests/import_files.rs @@ -0,0 +1,789 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `import files`: an account export read back, manifest and all, so what +//! the open formats can't hold comes back too. An archive exported and +//! imported again holds the same account, and exporting that gives the +//! same tree. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use inbuxa_migrate::db::{blobs, init}; +use inbuxa_migrate::export_files::{self, ExportFilesConfig}; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_dav::calcard; +use inbuxa_migrate::sync::import_files::{self, FilesImportConfig}; +use inbuxa_migrate::sync::{CommonConfig, Summary, TypeCounts}; +use rusqlite::{Connection, params}; +use serde_json::{Value, json}; +use tempfile::TempDir; + +const VCARD_ALICE: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-alice\r\nFN:Alice Smith\r\n\ +EMAIL:alice@example.com\r\nEND:VCARD\r\n"; +const VCARD_BOB: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-bob\r\nFN:Bob Jones\r\n\ +TEL:+1 555 0100\r\nEND:VCARD\r\n"; +const STANDUP: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\nBEGIN:VEVENT\r\nUID:ev-standup\r\n\ +DTSTAMP:20240101T000000Z\r\nDTSTART;TZID=Europe/Berlin:20240301T090000\r\nDURATION:PT15M\r\n\ +RRULE:FREQ=WEEKLY;COUNT=4\r\nSUMMARY:Standup\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; +const OFFSITE: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\nBEGIN:VEVENT\r\nUID:ev-offsite\r\n\ +DTSTAMP:20240101T000000Z\r\nDTSTART;VALUE=DATE:20240510\r\nSUMMARY:Offsite\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; + +fn message(subject: &str) -> Vec { + format!( + "From: dave@example.net\r\nTo: alice@example.org\r\nSubject: {subject}\r\n\ + Message-ID: <{}@example.net>\r\nDate: Fri, 01 Mar 2024 12:00:00 +0000\r\n\r\n{subject}\r\n", + subject.replace(' ', "-") + ) + .into_bytes() +} + +/// An account with everything the manifest exists for: folder names Maildir +/// can't hold, roles, an unsubscribed folder, a message in two folders, +/// custom keywords, default address book and calendar, a card and an event +/// in two collections, colors and time zones, an active Sieve script, +/// media types, a link, identities with signatures, a calendar identity. +fn seed(path: &Path) { + let conn = init::open(path).unwrap(); + conn.execute( + "INSERT INTO sources (id, kind, session_url, account_id, account_name, username) + VALUES (1, 'jmap', 'https://mail.example.org/jmap/session', 'a1', 'alice@example.org', 'alice')", + [], + ) + .unwrap(); + for (id, name, parent, role, subscribed) in [ + (1, "Inbox", None, Some("inbox"), 1), + (2, "Clients", Some(1), None, 1), + (3, "Archive", None, Some("archive"), 1), + (4, "2024", Some(3), None, 1), + (5, "Équipe & co", None, None, 1), + (6, "v1.0 notes", None, None, 0), + (7, "Sent Items", None, Some("sent"), 1), + ] { + conn.execute( + "INSERT INTO mailboxes (id, name, parent_id, role, is_subscribed) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![id, name, parent, role, subscribed], + ) + .unwrap(); + } + for (subject, mailboxes, keywords) in [ + ("Seen and flagged", "[1]", r#"["$flagged","$seen"]"#), + ("Junk in two folders", "[1,3]", r#"["$Junk"]"#), + ("Client note", "[2]", r#"["$answered","project-x"]"#), + ("Old report", "[4]", "[]"), + ("Bonjour", "[5]", r#"["$seen"]"#), + ("Release notes", "[6]", "[]"), + ("Sent reply", "[7]", r#"["$seen"]"#), + ] { + let blob = blobs::intern_blob(&conn, &message(subject)).unwrap(); + conn.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-03-01T12:00:00Z', ?2, ?3)", + params![blob, mailboxes, keywords], + ) + .unwrap(); + } + + conn.execute( + "INSERT INTO address_books (id, name, description, is_default) + VALUES (1, 'Personal', 'Friends and family', 0), (2, 'Work', NULL, 1)", + [], + ) + .unwrap(); + for (vcard, books) in [(VCARD_ALICE, "[1,2]"), (VCARD_BOB, "[2]")] { + let card = calcard::vcard_to_jscontact(vcard, "/x").unwrap(); + conn.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![card.uid, books, card.data.to_string()], + ) + .unwrap(); + } + + conn.execute( + "INSERT INTO calendars (id, name, description, color, time_zone, is_default) + VALUES (1, 'Team, Berlin', 'Stand-ups', '#3366ff', 'Europe/Berlin', 1), + (2, 'Company', NULL, '#ff9900', 'America/Phoenix', 0)", + [], + ) + .unwrap(); + for (ics, calendars) in [(STANDUP, "[1]"), (OFFSITE, "[1,2]")] { + let mut entry = calcard::ical_to_jscalendar_entries(ics).unwrap().remove(0); + calcard::strip_extracted_fields_from_event(&mut entry.data); + conn.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, ?2)", + params![calendars, entry.data.to_string()], + ) + .unwrap(); + } + + for (name, active, script) in [ + ( + "filters", + 1, + "require \"fileinto\";\r\nif header :contains \"subject\" \"report\" { fileinto \"Archive\"; }\r\n", + ), + ("old", 0, "keep;\r\n"), + ] { + let blob = blobs::intern_blob(&conn, script.as_bytes()).unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)", + params![name, active, blob], + ) + .unwrap(); + } + + let text = blobs::intern_blob(&conn, b"hello\n").unwrap(); + let pdf = blobs::intern_blob(&conn, b"%PDF-1.7 not really\n").unwrap(); + conn.execute( + "INSERT INTO file_nodes (id, parent_id, node_type, blob_id, target, name, media_type, created, modified) + VALUES (1, NULL, 'directory', NULL, NULL, 'Docs', NULL, '2024-01-01T00:00:00Z', NULL), + (2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'), + (3, 1, 'directory', NULL, NULL, 'Taxes', NULL, '2024-01-01T00:00:00Z', NULL), + (4, 3, 'file', ?2, NULL, '2025.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-04-15T09:30:00Z'), + (5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL)", + params![text, pdf], + ) + .unwrap(); + conn.execute( + "INSERT INTO identities (name, email, reply_to, bcc, text_signature, html_signature) + VALUES ('Alice Smith', 'alice@example.org', '[{\"name\":\"Alice\",\"email\":\"replies@example.org\"}]', + NULL, 'Alice\n-- \nExample Org', '

Alice

'), + ('Alice (support)', 'support@example.org', NULL, '[{\"name\":null,\"email\":\"log@example.org\"}]', '', '')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO participant_identities (name, calendar_address, is_default) + VALUES ('Alice Smith', 'mailto:alice@example.org', 1), + ('Alice (support)', 'mailto:support@example.org', 0)", + [], + ) + .unwrap(); +} + +struct Fixture { + _td: TempDir, + root: PathBuf, + archive: PathBuf, + out: PathBuf, +} + +fn exported() -> Fixture { + let td = tempfile::tempdir().unwrap(); + let root = td.path().to_path_buf(); + let archive = root.join("archive.sqlite"); + let out = root.join("out"); + seed(&archive); + export(&archive, &out); + Fixture { + _td: td, + root, + archive, + out, + } +} + +fn export(archive: &Path, out: &Path) { + let summary = export_files::run(ExportFilesConfig { + archive: archive.to_path_buf(), + dir: out.to_path_buf(), + objects: None, + dry_run: false, + logger: Logger::from_flags(true, 0), + }) + .unwrap(); + assert!(!summary.any_failed(), "{:?}", summary.per_type); +} + +fn common(archive: &Path, dry_run: bool) -> CommonConfig { + CommonConfig { + archive: archive.to_path_buf(), + threads: 1, + dry_run, + max_retries: 0, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + } +} + +fn import(dir: &Path, archive: &Path) -> Result { + import_files::run( + common(archive, false), + FilesImportConfig { + dir: dir.to_path_buf(), + allow_source_change: false, + }, + ) +} + +fn counts<'a>(summary: &'a Summary, type_name: &str) -> &'a TypeCounts { + &summary + .per_type + .iter() + .find(|(t, _)| *t == type_name) + .unwrap_or_else(|| panic!("no {type_name} in {:?}", summary.per_type)) + .1 +} + +fn rows(conn: &Connection, sql: &str) -> Vec> { + let mut stmt = conn.prepare(sql).unwrap(); + let n = stmt.column_count(); + stmt.query_map([], |r| { + (0..n) + .map(|i| { + Ok(match r.get::<_, rusqlite::types::Value>(i)? { + rusqlite::types::Value::Null => Value::Null, + rusqlite::types::Value::Integer(v) => json!(v), + rusqlite::types::Value::Real(v) => json!(v), + rusqlite::types::Value::Text(v) => json!(v), + rusqlite::types::Value::Blob(v) => json!(blake3::hash(&v).to_hex().to_string()), + }) + }) + .collect() + }) + .unwrap() + .map(Result::unwrap) + .collect() +} + +/// An archive's account in terms no row id appears in: every collection by +/// its names, every item by what it is and where it is. +fn account(path: &Path) -> BTreeMap<&'static str, BTreeSet> { + let conn = Connection::open(path).unwrap(); + let mut out: BTreeMap<&'static str, BTreeSet> = BTreeMap::new(); + + let boxes: BTreeMap)> = + rows(&conn, "SELECT id, name, parent_id FROM mailboxes") + .into_iter() + .map(|r| { + ( + r[0].as_i64().unwrap(), + (r[1].as_str().unwrap().to_owned(), r[2].as_i64()), + ) + }) + .collect(); + let path_of = |mut id: i64| { + let mut parts = Vec::new(); + loop { + let (name, parent) = &boxes[&id]; + parts.insert(0, name.clone()); + match parent { + Some(p) => id = *p, + None => break parts.join("/"), + } + } + }; + for r in rows(&conn, "SELECT id, role, is_subscribed FROM mailboxes") { + out.entry("mailboxes").or_default().insert(format!( + "{} role={} subscribed={}", + path_of(r[0].as_i64().unwrap()), + r[1], + r[2] + )); + } + let ids = |v: &Value| -> Vec { serde_json::from_str(v.as_str().unwrap()).unwrap() }; + for r in rows( + &conn, + "SELECT b.data, e.mailbox_ids, e.keywords, e.received_at FROM emails e JOIN blobs b ON b.id = e.blob_id", + ) { + let mut folders: Vec = ids(&r[1]).into_iter().map(path_of).collect(); + folders.sort(); + let mut keywords: Vec = serde_json::from_str(r[2].as_str().unwrap()).unwrap(); + keywords.sort(); + out.entry("emails").or_default().insert(format!( + "{} in {folders:?} keywords={keywords:?} received={}", + r[0], r[3] + )); + } + + let names = |sql: &str| -> BTreeMap { + rows(&conn, sql) + .into_iter() + .map(|r| (r[0].as_i64().unwrap(), r[1].as_str().unwrap().to_owned())) + .collect() + }; + let books = names("SELECT id, name FROM address_books"); + for r in rows( + &conn, + "SELECT name, description, is_default FROM address_books", + ) { + out.entry("address_books") + .or_default() + .insert(format!("{r:?}")); + } + for r in rows( + &conn, + "SELECT uid, address_book_ids, data FROM contact_cards", + ) { + let mut in_books: Vec<&str> = ids(&r[1]).iter().map(|b| books[b].as_str()).collect(); + in_books.sort(); + out.entry("cards") + .or_default() + .insert(format!("{} in {in_books:?}", r[0])); + } + let calendars = names("SELECT id, name FROM calendars"); + for r in rows( + &conn, + "SELECT name, description, color, time_zone, is_default FROM calendars", + ) { + out.entry("calendars").or_default().insert(format!("{r:?}")); + } + for r in rows( + &conn, + "SELECT calendar_ids, data, data_type FROM calendar_events", + ) { + let data: Value = serde_json::from_str(r[1].as_str().unwrap()).unwrap(); + let mut in_calendars: Vec<&str> = + ids(&r[0]).iter().map(|c| calendars[c].as_str()).collect(); + in_calendars.sort(); + out.entry("events").or_default().insert(format!( + "{} {} {} in {in_calendars:?}", + data["uid"], data["title"], r[2] + )); + } + for r in rows( + &conn, + "SELECT s.name, s.is_active, b.data FROM sieve_scripts s JOIN blobs b ON b.id = s.blob_id", + ) { + out.entry("sieve").or_default().insert(format!("{r:?}")); + } + let nodes = names("SELECT id, name FROM file_nodes"); + let parents: BTreeMap> = rows(&conn, "SELECT id, parent_id FROM file_nodes") + .into_iter() + .map(|r| (r[0].as_i64().unwrap(), r[1].as_i64())) + .collect(); + for r in rows( + &conn, + "SELECT n.id, n.node_type, n.media_type, n.target, b.data, n.modified + FROM file_nodes n LEFT JOIN blobs b ON b.id = n.blob_id", + ) { + let mut id = r[0].as_i64().unwrap(); + let mut path = nodes[&id].clone(); + while let Some(Some(p)) = parents.get(&id) { + path = format!("{}/{path}", nodes[p]); + id = *p; + } + // A link has no file whose time the export could set. + let modified = if r[1] == "symlink" { + Value::Null + } else { + r[5].clone() + }; + out.entry("files").or_default().insert(format!( + "{path} {} {} {} {} {modified}", + r[1], r[2], r[3], r[4] + )); + } + for r in rows( + &conn, + "SELECT name, email, reply_to, bcc, text_signature, html_signature FROM identities", + ) { + // Addresses are JSON; the same addresses may be written in any key order. + let mut r = r; + for v in &mut r[2..4] { + if let Some(text) = v.as_str() { + *v = serde_json::from_str(text).unwrap(); + } + } + out.entry("identities") + .or_default() + .insert(format!("{r:?}")); + } + for r in rows( + &conn, + "SELECT name, calendar_address, is_default FROM participant_identities", + ) { + out.entry("participant_identities") + .or_default() + .insert(format!("{r:?}")); + } + out +} + +#[test] +fn an_exported_account_comes_back_whole() { + let f = exported(); + let back = f.root.join("back.sqlite"); + let summary = import(&f.out, &back).unwrap(); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + + let original = account(&f.archive); + let read_back = account(&back); + for part in original.keys() { + assert_eq!(read_back.get(part), original.get(part), "{part}"); + } + assert_eq!(read_back, original); + + // One message in two folders came back as one message, not two. + assert_eq!(counts(&summary, "email").created, 7); + // Folders Maildir can't name kept their real names. + let mailboxes = &read_back["mailboxes"]; + assert!( + mailboxes.contains("v1.0 notes role=null subscribed=0"), + "{mailboxes:?}" + ); + assert!( + mailboxes.contains("Équipe & co role=null subscribed=1"), + "{mailboxes:?}" + ); + + let conn = Connection::open(&back).unwrap(); + let source: (String, String) = conn + .query_row("SELECT kind, account_name FROM sources", [], |r| { + Ok((r.get(0)?, r.get(1)?)) + }) + .unwrap(); + assert_eq!(source, ("files".to_owned(), "alice@example.org".to_owned())); +} + +#[test] +fn exporting_what_was_imported_gives_the_same_tree() { + let f = exported(); + let back = f.root.join("back.sqlite"); + import(&f.out, &back).unwrap(); + let again = f.root.join("again"); + export(&back, &again); + + let mut first = manifest(&f.out); + let mut second = manifest(&again); + for m in [&mut first, &mut second] { + let m = m.as_object_mut().unwrap(); + m.remove("created"); + m.remove("sources"); + } + assert_eq!(second, first); + assert_eq!(tree(&again), tree(&f.out)); +} + +/// Every file in an export by path, with its content; messages by folder +/// and content, since their names carry archive ids. +fn tree(root: &Path) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + stack.push(path); + continue; + } + let rel = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(); + if rel == "manifest.json" { + continue; + } + let bytes = fs::read(&path).unwrap(); + // iCalendar gives the order of a calendar's events no meaning. + let bytes = if rel.ends_with(".ics") { + components_sorted(&String::from_utf8(bytes).unwrap()).into_bytes() + } else { + bytes + }; + let content = blake3::hash(&bytes).to_hex().to_string(); + let key = match rel.split_once("/cur/") { + Some((folder, name)) => { + let flags = name.rsplit_once(":2,").map(|(_, f)| f).unwrap_or(""); + format!("{folder}/cur/{content}:2,{flags}") + } + None => rel, + }; + out.insert(key, content); + } + } + out +} + +/// An iCalendar object with its components in sorted order, each kept as +/// written. +fn components_sorted(ics: &str) -> String { + let mut head = String::new(); + let mut components: Vec = Vec::new(); + let mut current: Option = None; + for line in ics.split_inclusive('\n') { + let bare = line.trim_end(); + match &mut current { + None if bare.starts_with("BEGIN:") && bare != "BEGIN:VCALENDAR" => { + current = Some(line.to_owned()); + } + None => head.push_str(line), + Some(text) => { + text.push_str(line); + if bare.starts_with("END:") && text.starts_with(&format!("BEGIN:{}", &bare[4..])) { + components.push(current.take().unwrap()); + } + } + } + } + components.sort(); + head + &components.concat() +} + +fn manifest(out: &Path) -> Value { + serde_json::from_str(&fs::read_to_string(out.join("manifest.json")).unwrap()).unwrap() +} + +#[test] +fn a_second_run_finds_everything_in_place() { + let f = exported(); + let back = f.root.join("back.sqlite"); + import(&f.out, &back).unwrap(); + let before = account(&back); + let summary = import(&f.out, &back).unwrap(); + for (type_name, c) in &summary.per_type { + assert_eq!( + (c.created, c.updated, c.failed), + (0, 0, 0), + "{type_name}: {c:?}" + ); + } + assert_eq!(counts(&summary, "email").skipped, 7); + assert_eq!(account(&back), before); +} + +#[test] +fn a_changed_export_updates_the_archive_and_deletes_nothing() { + let f = exported(); + let back = f.root.join("back.sqlite"); + import(&f.out, &back).unwrap(); + + // Read in Dovecot since: one message in the archive folder is now seen. + let archive_cur = f.out.join("mail/.Archive/cur"); + let name = fs::read_dir(&archive_cur) + .unwrap() + .next() + .unwrap() + .unwrap() + .file_name() + .into_string() + .unwrap(); + fs::rename( + archive_cur.join(&name), + archive_cur.join(format!("{name}S")), + ) + .unwrap(); + // And the Sieve script that runs is now the other one. + let mut m = manifest(&f.out); + for script in m["sieve"].as_array_mut().unwrap() { + let active = script["name"] == "old"; + script["active"] = json!(active); + } + fs::write(f.out.join("manifest.json"), m.to_string()).unwrap(); + + let summary = import(&f.out, &back).unwrap(); + assert_eq!( + counts(&summary, "email").updated, + 1, + "{:?}", + summary.per_type + ); + assert_eq!(counts(&summary, "sievescript").updated, 2); + let conn = Connection::open(&back).unwrap(); + let active: String = conn + .query_row( + "SELECT name FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(active, "old"); + let junk: String = conn + .query_row( + "SELECT keywords FROM emails WHERE json_array_length(mailbox_ids) = 2", + [], + |r| r.get(0), + ) + .unwrap(); + // The copy in the inbox wasn't marked; the one in Archive was, and a + // message has one set of keywords. + assert_eq!(junk, r#"["$Junk","$seen"]"#); + let emails: i64 = conn + .query_row("SELECT count(*) FROM emails", [], |r| r.get(0)) + .unwrap(); + assert_eq!(emails, 7); +} + +#[test] +fn an_unfinished_export_or_a_later_version_is_refused() { + let f = exported(); + let back = f.root.join("back.sqlite"); + let text = fs::read_to_string(f.out.join("manifest.json")).unwrap(); + + fs::remove_file(f.out.join("manifest.json")).unwrap(); + let e = import(&f.out, &back).unwrap_err(); + assert_eq!(e.exit_code(), 1); + assert!(e.to_string().contains("interrupted"), "{e}"); + assert!( + !back.exists(), + "nothing is created for an export that can't be read" + ); + + let mut m: Value = serde_json::from_str(&text).unwrap(); + m["formatVersion"] = json!(2); + fs::write(f.out.join("manifest.json"), m.to_string()).unwrap(); + let e = import(&f.out, &back).unwrap_err(); + assert!(e.to_string().contains("version 2"), "{e}"); + + // What 27.1.0-rc.2 wrote is the same format under its old name. + m["formatVersion"] = json!(1); + m["format"] = json!("inbuxa-migrate-files"); + fs::write(f.out.join("manifest.json"), m.to_string()).unwrap(); + import(&f.out, &back).unwrap(); +} + +#[test] +fn a_dry_run_reads_the_export_and_writes_nothing() { + let f = exported(); + let back = f.root.join("back.sqlite"); + let summary = import_files::run( + common(&back, true), + FilesImportConfig { + dir: f.out.clone(), + allow_source_change: false, + }, + ) + .unwrap(); + assert!(!back.exists()); + assert_eq!(counts(&summary, "mailbox").fetched, 7); + // The message in two folders is a file in each. + assert_eq!(counts(&summary, "email").fetched, 8); + assert_eq!(counts(&summary, "contactcard").fetched, 3); + assert_eq!(counts(&summary, "calendarevent").fetched, 3); + assert_eq!(counts(&summary, "filenode").fetched, 5); +} + +#[test] +fn another_export_needs_allow_source_change() { + let f = exported(); + let back = f.root.join("back.sqlite"); + import(&f.out, &back).unwrap(); + let copy = f.root.join("copy"); + export(&back, ©); + let e = import(©, &back).unwrap_err(); + assert_eq!(e.exit_code(), 4, "{e}"); +} + +#[cfg(unix)] +#[test] +fn nothing_outside_the_export_is_read() { + use std::os::unix::fs::symlink; + + let f = exported(); + let outside = f.root.join("outside"); + fs::create_dir_all(outside.join("cur")).unwrap(); + fs::create_dir_all(outside.join("new")).unwrap(); + fs::create_dir_all(outside.join("tmp")).unwrap(); + let secret = b"Subject: not yours\r\n\r\nsecret\r\n"; + fs::write(outside.join("cur/1.x:2,S"), secret).unwrap(); + fs::write( + outside.join("secret.vcf"), + VCARD_BOB.replace("card-bob", "card-secret"), + ) + .unwrap(); + + let mut m = manifest(&f.out); + // Paths that climb out, by name and through links. + m["mail"]["folders"].as_array_mut().unwrap().push(json!({ + "name": "Escape", "path": ["Escape"], "dir": "mail/../outside", + "role": null, "subscribed": true, "messages": 1 + })); + symlink(&outside, f.out.join("mail/.Linked")).unwrap(); + m["mail"]["folders"].as_array_mut().unwrap().push(json!({ + "name": "Linked", "path": ["Linked"], "dir": "mail/.Linked", + "role": null, "subscribed": true, "messages": 1 + })); + symlink(outside.join("cur/1.x:2,S"), f.out.join("mail/cur/2.x:2,S")).unwrap(); + m["contacts"].as_array_mut().unwrap().push(json!({ + "name": "Stolen", "description": null, "file": "/etc/passwd", "default": false, "cards": 1 + })); + symlink( + outside.join("secret.vcf"), + f.out.join("contacts/Linked.vcf"), + ) + .unwrap(); + m["contacts"].as_array_mut().unwrap().push(json!({ + "name": "Linked", "description": null, "file": "contacts/Linked.vcf", "default": false, "cards": 1 + })); + symlink(&outside, f.out.join("files/Docs/elsewhere")).unwrap(); + m["files"]["links"].as_array_mut().unwrap().push(json!({ + "path": "files/../../x", "target": {"path": "/"} + })); + fs::write(f.out.join("manifest.json"), m.to_string()).unwrap(); + + let back = f.root.join("back.sqlite"); + let summary = import(&f.out, &back).unwrap(); + assert!( + counts(&summary, "email").failed >= 2, + "{:?}", + summary.per_type + ); + assert!(counts(&summary, "contactcard").failed >= 2); + assert!(counts(&summary, "filenode").failed >= 1); + + let conn = Connection::open(&back).unwrap(); + let leaked: i64 = conn + .query_row( + "SELECT count(*) FROM blobs WHERE instr(data, 'secret') > 0", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(leaked, 0); + let cards: i64 = conn + .query_row( + "SELECT count(*) FROM contact_cards WHERE uid = 'card-secret'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(cards, 0); + let elsewhere: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE name = 'elsewhere'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(elsewhere, 0); + // Everything that was safely inside still came in. + assert_eq!(counts(&summary, "email").created, 7); +} + +#[test] +fn a_card_another_import_put_in_the_archive_keeps_its_books() { + let f = exported(); + let back = f.root.join("back.sqlite"); + { + let conn = init::open(&back).unwrap(); + conn.execute( + "INSERT INTO sources (id, kind, session_url, account_id, account_name, username) + VALUES (1, 'carddav', 'https://dav.example.org/', 'alice', 'alice', 'alice')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO address_books (id, name) VALUES (1, 'Phone')", + [], + ) + .unwrap(); + let card = calcard::vcard_to_jscontact(VCARD_ALICE, "/x").unwrap(); + conn.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, '[1]', ?2)", + params![card.uid, card.data.to_string()], + ) + .unwrap(); + } + import(&f.out, &back).unwrap(); + let cards = &account(&back)["cards"]; + assert!( + cards.contains(r#""card-alice" in ["Personal", "Phone", "Work"]"#), + "{cards:?}" + ); +} -- 2.54.0