diff --git a/.gitea/workflows/upstream-watch.yml b/.gitea/workflows/upstream-watch.yml index a7aca19..b563375 100644 --- a/.gitea/workflows/upstream-watch.yml +++ b/.gitea/workflows/upstream-watch.yml @@ -81,7 +81,7 @@ jobs: "git fetch origin upstream:upstream\n" f"scripts/upstream-import.py {tag}\n" f"git switch -c import/upstream-{tag} origin/main\n" - "git merge upstream\n" + "git merge refs/heads/upstream\n" "```\n\n" "Before reading the changes, scan them for hidden characters and text aimed at " "automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.") diff --git a/CHANGELOG.md b/CHANGELOG.md index c8acae8..922bd8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,41 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +Upstream's 1.0.12 and 1.0.13 are merged in. + +### Added +- The out-of-office reply comes across: imported from a server's + `VacationResponse`, and exported to the target's, in place of copying the + script the server keeps for it. +- WebDAV, CalDAV, CardDAV: `--replace-source` removes the other recorded + sources of the same kind, and what was imported from them, before + importing, so a moved share leaves one tree in the archive. +- Files keep their `executable` flag where the server has one. An archive + gains the column at its next import; until then `inspect` and the other + read-only commands call it an older layout. + +### Changed +- FileNodes are read in either shape servers use: `nodeType` and `type`, as + inbuxa and other 0.16-line servers spell them, or `@type` and `mediaType` + (draft-ietf-jmap-filenode-18, as in Stalwart 1.0). An export asks the + target which it takes and writes that one. Upstream dropped the older + shape; this fork keeps both. +- The Graph device-code sign-in asks for `Files.Read`, so the default run + reaches OneDrive. +- iCalendar and vCard parse failures say what was wrong and quote the line. + +### Fixed +- IMAP: children of a `\Noselect` parent folder kept their place instead of + landing at the top level. +- JMAP import no longer holds every blob of a page in memory at once, which + large messages could exhaust. +- CalDAV, CardDAV: items with bare CR line breaks, as Kerio Connect sends, + are read. +- EWS: a bare `--auth-bearer` takes the API key or token from the + environment without asking for `--ews-client-id`. +- `--account-name` as an administrator takes the data account from the + principal's `accounts`, as RFC 9670 defines it. + ## [27.1-rc.6] The sixth release candidate of the 27.1 series: whole domains from Microsoft diff --git a/Cargo.lock b/Cargo.lock index 6b35422..dae0402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1337,6 +1337,7 @@ dependencies = [ "encodify", "flate2", "getrandom 0.3.4", + "hashify", "indexmap 2.14.2", "mail-parser", "mockito", @@ -1544,9 +1545,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.38.2" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3" dependencies = [ "cc", "pkg-config", @@ -2130,9 +2131,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "da9bba6f3506135e120a6586df05be18b48d18896600646a4bec8f993d91a594" dependencies = [ "bitflags 2.13.2", "fallible-iterator", diff --git a/Cargo.toml b/Cargo.toml index 2793465..6f1db5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ bundled-sqlite = ["rusqlite/bundled"] serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0" indexmap = { version = "2.14", features = ["serde"] } -rusqlite = { version = "0.40", features = ["blob"] } +rusqlite = { version = "0.41", features = ["blob"] } blake3 = "1.8.5" time = { version = "0.3.47", features = ["serde-well-known", "macros"] } url= "2.5.8" @@ -40,6 +40,7 @@ flate2 = "1.1.9" quick-xml = "0.42" chrono = { version = "0.4", default-features = false, features = ["std"] } chrono-tz = { version = "0.10", default-features = false } +hashify = "0.2" psl = "2.1.226" getrandom = "0.3.4" diff --git a/docs/usage.md b/docs/usage.md index e54c1fe..2a83d37 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -59,7 +59,10 @@ inbuxa-migrate import [source-args...] Reads a source account into the SQLite `ARCHIVE`, creating it if it is absent. An archive remembers which account filled it; every importer takes -`--allow-source-change` to fill it from a different one anyway. +`--allow-source-change` to fill it from a different one anyway. The DAV +importers (`caldav`, `carddav`, `webdav`) also take `--replace-source`, which +first removes the other sources of the same kind, and what came from them, so +a share that moved leaves one tree in the archive, not two. ### JMAP @@ -74,6 +77,11 @@ inbuxa-migrate import jmap \ Imports one JMAP account, from inbuxa, Stalwart or any other JMAP server. `--objects` accepts a comma-separated list of object tokens (`mailbox,email,calendar,calendarevent,addressbook,contactcard,identity,sievescript,participantidentity,filenode`). The default is everything the server advertises. +The out-of-office reply, the server's `VacationResponse`, comes with +`sievescript`. Files are read in either shape servers spell them: `nodeType` +and `type`, as inbuxa and other servers on the 0.16 line do, or `@type` and +`mediaType`, as draft-ietf-jmap-filenode-18 servers such as Stalwart 1.0 do. + ### IMAP ``` @@ -285,7 +293,11 @@ what changed at the source in between. `updated` time, the archive's is written only if it is newer; otherwise the properties that differ are written. - **Sieve scripts** are matched by name, and the target's is replaced when - its content differs from the archive's. + its content differs from the archive's. Where the target has a + `VacationResponse`, the out-of-office reply is written there, and the + script a server keeps for it is not copied as a script. +- **Files** are written in the shape the target takes, which `export` asks + it once per run: inbuxa takes `nodeType` and `type`. Messages go in batches, as many to an `Email/import` as the target's `maxObjectsInSet` allows, up to 50, with their blobs uploaded several at a diff --git a/scripts/upstream-import.py b/scripts/upstream-import.py index 7f27c2d..d2309e0 100755 --- a/scripts/upstream-import.py +++ b/scripts/upstream-import.py @@ -22,12 +22,13 @@ # # --first starts the branch. Give it the upstream commit the fork was made # from; the snapshot's parent is that commit. Merge it into main once with -# `git merge -s ours upstream`, which records that main already has it, and -# later imports merge cleanly against it. +# `git merge -s ours refs/heads/upstream`, which records that main already +# has it, and later imports merge cleanly against it. Name the branch in +# full: plain `upstream` is also the remote. # # After an import: # git switch -c import/upstream-REF origin/main -# git merge upstream +# git merge refs/heads/upstream # (resolve; for Cargo.lock take main's and let `cargo build` add what's new) # then the usual checks and a PR. Push `upstream` to origin only once that PR # has landed. The upstream-watch workflow reads the imported release from the diff --git a/src/cli.rs b/src/cli.rs index ecfa4b4..25425e9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -27,7 +27,7 @@ use crate::secret; use crate::sync::export_dav::ExportDavConfig; use crate::sync::export_imap::ExportImapConfig; use crate::sync::export_managesieve::ExportManageSieveConfig; -use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig}; use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; use crate::sync::import_imap::{ImapAuth, ImapImportConfig}; @@ -920,6 +920,13 @@ pub struct DavImportArgs { )] allow_source_change: bool, + #[arg( + long, + conflicts_with = "allow_source_change", + help = "Remove other recorded sources of this kind, and their items, before importing" + )] + replace_source: bool, + #[command(flatten)] global: GlobalArgs, @@ -1159,7 +1166,11 @@ fn resolve_dav_import(args: DavImportArgs, kind: DavKindArg) -> Result SourceChange::Replace, + (false, true) => SourceChange::Allow, + (false, false) => SourceChange::Reject, + }, }, )) } @@ -1648,38 +1659,43 @@ fn resolve_exchange_ews_auth(args: &ExchangeEwsImportArgs) -> Result Result { + let mut argv = vec!["inbuxa-migrate", "import", "exchange-ews", "archive.sqlite"]; + argv.extend_from_slice(extra); + let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}")); + match cli.resolve()? { + Action::ImportExchangeEws(_, config) => Ok(config.auth), + _ => panic!("expected ImportExchangeEws"), + } + } + + fn set_ews_env(token: Option<&str>, client_secret: Option<&str>) { + unsafe { + match token { + Some(t) => env::set_var("INBUXA_MIGRATE_TOKEN", t), + None => env::remove_var("INBUXA_MIGRATE_TOKEN"), + } + match client_secret { + Some(s) => env::set_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET", s), + None => env::remove_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET"), + } + } + } + + // A bare --auth-bearer takes INBUXA_MIGRATE_TOKEN without needing --ews-client-id. + #[test] + fn ews_bare_bearer_uses_env_token_without_client_id() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&["--auth-bearer"]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); + } + + // INBUXA_MIGRATE_TOKEN wins over client credentials taken from the environment. + #[test] + fn ews_env_token_beats_env_client_secret() { + let _g = lock(); + set_ews_env(Some("envtok"), Some("sec")); + let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); + } + + // The env client secret is used when no token is exported. + #[test] + fn ews_env_client_secret_without_token_uses_client_credentials() { + let _g = lock(); + set_ews_env(None, Some("sec")); + let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); + set_ews_env(None, None); + assert!(matches!( + auth, + Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { client_id, client_secret, .. })) + if client_id == "cid" && client_secret == "sec" + )); + } + + // An explicit --ews-device-code is not overridden by an exported token. + #[test] + fn ews_device_code_flag_beats_env_token() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&[ + "--auth-bearer", + "--ews-client-id", + "cid", + "--ews-device-code", + ]); + set_ews_env(None, None); + assert!(matches!( + auth, + Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id })) + if tenant == "common" && client_id == "cid" + )); + } + + // --ews-device-code still needs a client id. + #[test] + fn ews_device_code_without_client_id_is_usage_error() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&["--auth-bearer", "--ews-device-code"]); + set_ews_env(None, None); + match auth { + Err(Error::Usage(m)) => assert!(m.contains("--ews-client-id"), "msg was: {m}"), + other => panic!("expected Usage error, got {other:?}"), + } + } + + // An inline token wins over everything else. + #[test] + fn ews_inline_token_beats_env_and_flows() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&[ + "--auth-bearer=inline", + "--ews-client-id", + "cid", + "--ews-device-code", + ]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "inline")); + } + fn graph_config(objects: &[&str]) -> Result { let mut argv = vec![ "inbuxa-migrate", @@ -2162,4 +2282,39 @@ mod tests { _ => panic!("a missing key file is a usage error"), } } + + fn webdav_source_change(extra: &[&str]) -> Result { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "webdav", + "--url", + "https://dav.example/files/u/", + "--auth-bearer", + "tok", + ]; + argv.extend_from_slice(extra); + argv.push("archive.sqlite"); + let cli = Cli::try_parse_from(argv)?; + match cli.resolve() { + Ok(Action::ImportDav(_, config)) => Ok(config.source_change), + _ => panic!("expected ImportDav"), + } + } + + // --replace-source maps to Replace and cannot be combined with --allow-source-change. + #[test] + fn dav_source_change_flags_resolve() { + let _g = lock(); + assert_eq!(webdav_source_change(&[]).unwrap(), SourceChange::Reject); + assert_eq!( + webdav_source_change(&["--allow-source-change"]).unwrap(), + SourceChange::Allow + ); + assert_eq!( + webdav_source_change(&["--replace-source"]).unwrap(), + SourceChange::Replace + ); + assert!(webdav_source_change(&["--replace-source", "--allow-source-change"]).is_err()); + } } diff --git a/src/db/init.rs b/src/db/init.rs index 7baa167..f25ed96 100644 --- a/src/db/init.rs +++ b/src/db/init.rs @@ -70,10 +70,17 @@ pub fn open_read_only(path: &std::path::Path) -> Result { let missing: Vec<&str> = schema_tables() .filter(|t| !present.iter().any(|p| p == t)) .collect(); - let has_data_type = conn - .prepare("SELECT 1 FROM pragma_table_info('calendar_events') WHERE name = 'data_type'")? - .exists([])?; - if !missing.is_empty() || !has_data_type { + // Columns added to tables older archives already have. + let mut has_columns = true; + for (table, column) in [ + ("calendar_events", "data_type"), + ("file_nodes", "executable"), + ] { + has_columns &= conn + .prepare("SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2")? + .exists([table, column])?; + } + if !missing.is_empty() || !has_columns { return Err(OpenError::NotAnArchive(format!( "{} was written by an older inbuxa-migrate and lacks part of today's layout; \ the next import into it brings it up to date", @@ -122,6 +129,7 @@ pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> { let tx = conn.unchecked_transaction()?; tx.execute_batch(SCHEMA_SQL)?; ensure_calendar_events_data_type(&tx)?; + ensure_file_nodes_executable(&tx)?; ensure_graph_ids_accept_file_nodes(&tx)?; ensure_dav_ids_share_local_rows(&tx)?; tx.commit()?; @@ -129,14 +137,34 @@ pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> { } fn ensure_calendar_events_data_type(conn: &Connection) -> Result<(), OpenError> { - let mut stmt = conn.prepare("PRAGMA table_info(calendar_events)")?; + ensure_column( + conn, + "calendar_events", + "data_type", + "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", + ) +} + +fn ensure_file_nodes_executable(conn: &Connection) -> Result<(), OpenError> { + ensure_column( + conn, + "file_nodes", + "executable", + "ALTER TABLE file_nodes ADD COLUMN executable INTEGER NOT NULL DEFAULT 0", + ) +} + +fn ensure_column( + conn: &Connection, + table: &str, + column: &str, + alter: &str, +) -> Result<(), OpenError> { + let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; - let has_column = rows.filter_map(|r| r.ok()).any(|name| name == "data_type"); + let has_column = rows.filter_map(|r| r.ok()).any(|name| name == column); if !has_column { - conn.execute( - "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", - [], - )?; + conn.execute(alter, [])?; } Ok(()) } diff --git a/src/db/mod.rs b/src/db/mod.rs index 14a313a..351d35a 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -20,3 +20,4 @@ pub mod roles; pub mod sources; pub mod sync_state_jmap; pub mod takeout_ids; +pub mod vacation_response; diff --git a/src/db/schema.sql b/src/db/schema.sql index b0674af..6603294 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -172,6 +172,17 @@ CREATE TABLE IF NOT EXISTS sieve_scripts ( CREATE UNIQUE INDEX IF NOT EXISTS sieve_scripts_one_active_idx ON sieve_scripts (is_active) WHERE is_active = 1; +CREATE TABLE IF NOT EXISTS vacation_response ( + id INTEGER PRIMARY KEY CHECK (id = 1), + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + is_enabled INTEGER NOT NULL DEFAULT 0, + from_date TEXT, + to_date TEXT, + subject TEXT, + text_body TEXT, + html_body TEXT +); + CREATE TABLE IF NOT EXISTS address_books ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, @@ -236,6 +247,7 @@ CREATE TABLE IF NOT EXISTS file_nodes ( modified TEXT, is_subscribed INTEGER NOT NULL DEFAULT 1, role TEXT, + executable INTEGER NOT NULL DEFAULT 0, UNIQUE (parent_id, name) ); diff --git a/src/db/sources.rs b/src/db/sources.rs index 8a2600a..f55bd9b 100644 --- a/src/db/sources.rs +++ b/src/db/sources.rs @@ -37,6 +37,27 @@ pub fn conflicting_source( .optional() } +pub fn other_sources( + conn: &Connection, + kind: &str, + session_url: &str, + account_id: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT id, session_url, account_id FROM sources + WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) ORDER BY id", + )?; + stmt.query_map(params![kind, session_url, account_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .collect() +} + +pub fn delete_source(conn: &Connection, id: i64) -> Result<(), rusqlite::Error> { + conn.execute("DELETE FROM sources WHERE id = ?1", params![id])?; + Ok(()) +} + pub fn upsert_source( conn: &Connection, key: &SourceKey, @@ -102,4 +123,20 @@ mod tests { .unwrap(); assert_eq!(conflict, ("https://a/jmap".to_owned(), "w".to_owned())); } + + #[test] + fn other_sources_lists_every_other_source_of_the_kind() { + let c = mem(); + let a = upsert_source(&c, &key("https://a/jmap", "w"), None, "alice").unwrap(); + let b = upsert_source(&c, &key("https://b/jmap", "w"), None, "alice").unwrap(); + upsert_source(&c, &key("https://c/jmap", "w"), None, "alice").unwrap(); + let others = other_sources(&c, "jmap", "https://c/jmap", "w").unwrap(); + let ids: Vec = others.iter().map(|(id, _, _)| *id).collect(); + assert_eq!(ids, [a, b]); + assert!( + other_sources(&c, "imap", "https://c/jmap", "w") + .unwrap() + .is_empty() + ); + } } diff --git a/src/db/vacation_response.rs b/src/db/vacation_response.rs new file mode 100644 index 0000000..18a6607 --- /dev/null +++ b/src/db/vacation_response.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::jmap::wire::vacation_response::VacationResponse; + +pub fn store( + conn: &Connection, + source_id: i64, + vacation: &VacationResponse, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO vacation_response + (id, source_id, is_enabled, from_date, to_date, subject, text_body, html_body) + VALUES (1, ?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + source_id, + vacation.is_enabled, + vacation.from_date, + vacation.to_date, + vacation.subject, + vacation.text_body, + vacation.html_body, + ], + )?; + Ok(()) +} + +pub fn clear(conn: &Connection, source_id: i64) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM vacation_response WHERE source_id = ?1", + params![source_id], + )?; + Ok(()) +} + +pub fn exists(conn: &Connection) -> Result { + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM vacation_response)", + [], + |row| row.get(0), + ) +} + +pub fn load(conn: &Connection) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT is_enabled, from_date, to_date, subject, text_body, html_body + FROM vacation_response WHERE id = 1", + [], + |row| { + Ok(VacationResponse { + is_enabled: row.get(0)?, + from_date: row.get(1)?, + to_date: row.get(2)?, + subject: row.get(3)?, + text_body: row.get(4)?, + html_body: row.get(5)?, + }) + }, + ) + .optional() +} diff --git a/src/exchange_graph/oauth.rs b/src/exchange_graph/oauth.rs index 458ff66..c599589 100644 --- a/src/exchange_graph/oauth.rs +++ b/src/exchange_graph/oauth.rs @@ -15,8 +15,7 @@ use ureq::config::Config; use crate::exchange_graph::error::GraphError; use crate::net::{tls, with_timeouts}; -pub const SCOPES: &str = - "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read"; +pub const SCOPES: &str = "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read Files.Read"; /// App-only: the application permissions an administrator granted the app. pub const APP_ONLY_SCOPE: &str = "https://graph.microsoft.com/.default"; @@ -462,6 +461,24 @@ mod tests { format!("{header}.{payload}.") } + // Device code consent covers every --objects surface, including the drive. + #[test] + fn scopes_cover_every_surface() { + for required in [ + "offline_access", + "Mail.Read", + "MailboxSettings.Read", + "Calendars.Read", + "Contacts.Read", + "Files.Read", + ] { + assert!( + SCOPES.split_whitespace().any(|s| s == required), + "missing {required}" + ); + } + } + #[test] fn jwt_claim_extraction() { let token = make_jwt("t-1", "alice@x.com", 9999999999); diff --git a/src/jmap/account.rs b/src/jmap/account.rs index 4ecb95a..8ec02d9 100644 --- a/src/jmap/account.rs +++ b/src/jmap/account.rs @@ -19,7 +19,6 @@ pub enum AccountSelector { Name(String), } -const OWNER_URN: &str = "urn:ietf:params:jmap:principals:owner"; const PRINCIPALS_URN: &str = "urn:ietf:params:jmap:principals"; pub fn resolve( @@ -140,21 +139,15 @@ fn unsupported_principals(name: &str, session: &Session) -> Error { fn extract_account_id(principal: &Value, name: &str) -> Result { if let Some(accounts) = principal.get("accounts").and_then(Value::as_object) { - for data in accounts.values() { - if let Some(owner) = data.get(OWNER_URN) - && let Some(id) = owner.get("accountIdForPrincipal").and_then(Value::as_str) - { - return Ok(id.to_owned()); - } + let mut keys = accounts.keys(); + if let (Some(only), None) = (keys.next(), keys.next()) { + return Ok(only.clone()); } - for data in accounts.values() { - if let Some(cap) = data.as_object() { - for v in cap.values() { - if let Some(id) = v.get("accountId").and_then(Value::as_str) { - return Ok(id.to_owned()); - } - } - } + let named = accounts + .iter() + .find(|(_, account)| account.get("name").and_then(Value::as_str) == Some(name)); + if let Some((id, _)) = named { + return Ok(id.clone()); } } principal @@ -256,12 +249,33 @@ mod tests { assert_eq!(extract_account_id(&p, "bob").unwrap(), "acc-7"); } + // RFC 9670: the data account is the key of "accounts", not accountIdForPrincipal. #[test] - fn extract_uses_per_capability_account_id() { + fn extract_uses_accounts_key_not_account_id_for_principal() { + let p = json!({ + "id": "px", + "name": "bob@example.org", + "accounts": { "data-9": { + "name": "bob@example.org", + "isPersonal": false, + "accountCapabilities": { + "urn:ietf:params:jmap:principals:owner": + { "accountIdForPrincipal": "admin-1", "principalId": "px" } + } + } } + }); + assert_eq!(extract_account_id(&p, "bob@example.org").unwrap(), "data-9"); + } + + #[test] + fn extract_picks_the_named_account_among_several() { let p = json!({ "id": "px", "name": "bob", - "accounts": { "z": { "urn:ietf:params:jmap:mail": { "accountId": "data-9" } } } + "accounts": { + "shared-1": { "name": "team" }, + "data-9": { "name": "bob" } + } }); assert_eq!(extract_account_id(&p, "bob").unwrap(), "data-9"); } diff --git a/src/jmap/request.rs b/src/jmap/request.rs index 0a88ac6..74b86f6 100644 --- a/src/jmap/request.rs +++ b/src/jmap/request.rs @@ -18,6 +18,8 @@ use crate::logging::Logger; pub const URN_CORE: &str = "urn:ietf:params:jmap:core"; +pub const URN_VACATION_RESPONSE: &str = "urn:ietf:params:jmap:vacationresponse"; + /// The session capability every inbuxa server has. pub const URN_INBUXA: &str = "urn:inbuxa:jmap"; @@ -50,6 +52,7 @@ pub fn using_urn(method: &str, registry: Option<&'static str>) -> Result<&'stati "Calendar" | "CalendarEvent" | "ParticipantIdentity" => "urn:ietf:params:jmap:calendars", "FileNode" => "urn:ietf:params:jmap:filenode", "Principal" => "urn:ietf:params:jmap:principals", + "VacationResponse" => URN_VACATION_RESPONSE, _ => URN_CORE, }) } diff --git a/src/jmap/session.rs b/src/jmap/session.rs index 324197d..8058122 100644 --- a/src/jmap/session.rs +++ b/src/jmap/session.rs @@ -278,6 +278,13 @@ impl Session { .unwrap_or(false) } + pub fn may_support(&self, account_id: &str, capability_urn: &str) -> bool { + match self.account_capabilities(account_id) { + Some(caps) => caps.contains_key(capability_urn), + None => self.capabilities.contains_key(capability_urn), + } + } + pub fn upload_url_for(&self, account_id: &str) -> String { self.upload_url .replace("{accountId}", &encode_segment(account_id)) @@ -414,6 +421,16 @@ mod tests { assert!(!session.supports("missing", "urn:ietf:params:jmap:mail")); } + // Listed accounts use their own capabilities; unlisted ones fall back to the session's. + #[test] + fn may_support_falls_back_to_session_capabilities_for_unlisted_account() { + let session = parse_session(raw_session()).unwrap(); + assert!(session.may_support("w", "urn:ietf:params:jmap:mail")); + assert!(!session.may_support("w", "urn:ietf:params:jmap:core")); + assert!(session.may_support("missing", "urn:ietf:params:jmap:core")); + assert!(!session.may_support("missing", "urn:ietf:params:jmap:mail")); + } + #[test] fn templates_upload_and_download_urls_with_encoding() { let session = parse_session(raw_session()).unwrap(); diff --git a/src/jmap/wire/calendar.rs b/src/jmap/wire/calendar.rs index d7cd7dc..d59b658 100644 --- a/src/jmap/wire/calendar.rs +++ b/src/jmap/wire/calendar.rs @@ -49,6 +49,183 @@ pub struct Calendar { pub time_zone: Option, } +impl Calendar { + pub fn normalize_color(&mut self) -> Option { + let color = self.color.as_mut()?; + let trimmed = color.trim(); + let kept = match trimmed.strip_prefix('#') { + Some(hex) if hex.bytes().all(|byte| byte.is_ascii_hexdigit()) => match hex.len() { + 3 | 4 => trimmed.get(..=3), + 6 | 8 => trimmed.get(..=6), + _ => None, + }, + Some(_) => None, + None => is_css3_color_name(trimmed).then_some(trimmed), + }; + match kept { + Some(kept) if kept.len() == color.len() => None, + Some(kept) => { + *color = kept.to_owned(); + None + } + None => self.color.take(), + } + } +} + +fn is_css3_color_name(name: &str) -> bool { + hashify::set_ignore_case!( + name.as_bytes(), + "aliceblue", + "antiquewhite", + "aqua", + "aquamarine", + "azure", + "beige", + "bisque", + "black", + "blanchedalmond", + "blue", + "blueviolet", + "brown", + "burlywood", + "cadetblue", + "chartreuse", + "chocolate", + "coral", + "cornflowerblue", + "cornsilk", + "crimson", + "cyan", + "darkblue", + "darkcyan", + "darkgoldenrod", + "darkgray", + "darkgreen", + "darkgrey", + "darkkhaki", + "darkmagenta", + "darkolivegreen", + "darkorange", + "darkorchid", + "darkred", + "darksalmon", + "darkseagreen", + "darkslateblue", + "darkslategray", + "darkslategrey", + "darkturquoise", + "darkviolet", + "deeppink", + "deepskyblue", + "dimgray", + "dimgrey", + "dodgerblue", + "firebrick", + "floralwhite", + "forestgreen", + "fuchsia", + "gainsboro", + "ghostwhite", + "gold", + "goldenrod", + "gray", + "green", + "greenyellow", + "grey", + "honeydew", + "hotpink", + "indianred", + "indigo", + "ivory", + "khaki", + "lavender", + "lavenderblush", + "lawngreen", + "lemonchiffon", + "lightblue", + "lightcoral", + "lightcyan", + "lightgoldenrodyellow", + "lightgray", + "lightgreen", + "lightgrey", + "lightpink", + "lightsalmon", + "lightseagreen", + "lightskyblue", + "lightslategray", + "lightslategrey", + "lightsteelblue", + "lightyellow", + "lime", + "limegreen", + "linen", + "magenta", + "maroon", + "mediumaquamarine", + "mediumblue", + "mediumorchid", + "mediumpurple", + "mediumseagreen", + "mediumslateblue", + "mediumspringgreen", + "mediumturquoise", + "mediumvioletred", + "midnightblue", + "mintcream", + "mistyrose", + "moccasin", + "navajowhite", + "navy", + "oldlace", + "olive", + "olivedrab", + "orange", + "orangered", + "orchid", + "palegoldenrod", + "palegreen", + "paleturquoise", + "palevioletred", + "papayawhip", + "peachpuff", + "peru", + "pink", + "plum", + "powderblue", + "purple", + "red", + "rosybrown", + "royalblue", + "saddlebrown", + "salmon", + "sandybrown", + "seagreen", + "seashell", + "sienna", + "silver", + "skyblue", + "slateblue", + "slategray", + "slategrey", + "snow", + "springgreen", + "steelblue", + "tan", + "teal", + "thistle", + "tomato", + "turquoise", + "violet", + "wheat", + "white", + "whitesmoke", + "yellow", + "yellowgreen", + ) +} + fn default_true() -> bool { true } @@ -56,3 +233,56 @@ fn default_true() -> bool { fn default_include_in_availability() -> String { "all".to_owned() } + +#[cfg(test)] +mod tests { + use super::*; + + fn normalized(color: &str) -> (Option, Option) { + let mut calendar: Calendar = + serde_json::from_value(serde_json::json!({ "name": "C", "color": color })) + .expect("calendar"); + let dropped = calendar.normalize_color(); + (calendar.color, dropped) + } + + // Alpha is stripped, valid colors pass through, anything else is dropped. + #[test] + fn calendar_colors_are_normalized_for_jmap() { + for (input, expected) in [ + ("#AAAAAAFF", "#AAAAAA"), + ("#3366ccfe", "#3366cc"), + ("#ABCD", "#ABC"), + ("#abc", "#abc"), + ("#3366CC", "#3366CC"), + (" #3366cc\n", "#3366cc"), + ("CornflowerBlue", "CornflowerBlue"), + ("grey", "grey"), + ] { + assert_eq!( + normalized(input), + (Some(expected.to_owned()), None), + "{input:?}" + ); + } + for input in [ + "", + "#", + "#12345", + "#GGGGGG", + "#1234567", + "rgb(1,2,3)", + "notacolor", + ] { + assert_eq!( + normalized(input), + (None, Some(input.to_owned())), + "{input:?}" + ); + } + let mut calendar: Calendar = + serde_json::from_value(serde_json::json!({ "name": "C" })).expect("calendar"); + assert_eq!(calendar.normalize_color(), None); + assert_eq!(calendar.color, None); + } +} diff --git a/src/jmap/wire/file_node.rs b/src/jmap/wire/file_node.rs index d70acdf..8f2c515 100644 --- a/src/jmap/wire/file_node.rs +++ b/src/jmap/wire/file_node.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -7,9 +8,66 @@ use super::common::bool_or_true; use super::{JmapId, UtcDate}; use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// How a server spells a FileNode. Servers on the 0.16 line, inbuxa among +/// them, name the kind `nodeType` (`file`, `directory`, `symlink`) and a +/// file's media type `type`. draft-ietf-jmap-filenode-18, which Stalwart 1.0 +/// follows, says `@type` (`File`, `Directory`, `SymLink`) and `mediaType`, +/// and adds `executable`. Each refuses the other's names on a write and +/// ignores them on a get, so a get asks for both and a write uses the +/// server's own. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Shape { + Legacy, + Draft18, +} + +/// A FileNode, in either shape, read into this one; a node with neither +/// `@type` nor `nodeType` is a file if it has a blob, a symlink if it has a +/// target, and a directory otherwise. +pub fn to_draft18(node: &mut Map) { + let kind = match node.remove("@type") { + Some(Value::String(t)) => t, + _ => match node.remove("nodeType") { + Some(Value::String(t)) => t, + _ if node.get("blobId").is_some_and(|v| !v.is_null()) => "file".to_owned(), + _ if node.get("target").is_some_and(|v| !v.is_null()) => "symlink".to_owned(), + _ => "directory".to_owned(), + }, + }; + node.remove("nodeType"); + let kind = match kind.to_ascii_lowercase().as_str() { + "file" => "File".to_owned(), + "directory" => "Directory".to_owned(), + "symlink" => "SymLink".to_owned(), + _ => kind, + }; + node.insert("@type".to_owned(), Value::String(kind)); + if let Some(media_type) = node.remove("type") { + node.entry("mediaType").or_insert(media_type); + } +} + +/// A FileNode, or a patch to one, written as `shape` spells it. +pub fn for_server(node: &mut Map, shape: Shape) { + if shape == Shape::Draft18 { + return; + } + if let Some(Value::String(kind)) = node.remove("@type") { + node.insert( + "nodeType".to_owned(), + Value::String(kind.to_ascii_lowercase()), + ); + } + if let Some(media_type) = node.remove("mediaType") { + node.insert("type".to_owned(), media_type); + } + node.remove("executable"); +} #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", try_from = "Map")] pub struct FileNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub id: Option, @@ -17,20 +75,8 @@ pub struct FileNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub parent_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub node_type: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub blob_id: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub target: Option>, - pub name: String, - #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] - pub media_type: Option, - #[serde(with = "time::serde::rfc3339")] pub created: UtcDate, @@ -41,32 +87,155 @@ pub struct FileNode { )] pub modified: Option, - #[serde(default = "default_true", deserialize_with = "bool_or_true")] - pub is_subscribed: bool, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub role: Option, + #[serde(flatten)] + pub kind: NodeKind, } -impl FileNode { - pub fn effective_node_type(&self) -> NodeType { - match self.node_type { - Some(t) => t, - None if self.blob_id.is_some() => NodeType::File, - None if self.target.is_some() => NodeType::Symlink, - None => NodeType::Directory, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "@type")] +pub enum NodeKind { + File(FileProps), + Directory(DirectoryProps), + SymLink(SymLinkProps), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileProps { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob_id: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + + #[serde(default, deserialize_with = "bool_or_false")] + pub executable: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DirectoryProps { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + + #[serde(default = "default_true", deserialize_with = "bool_or_true")] + pub is_subscribed: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SymLinkProps { + pub target: Vec, +} + +/// `FileNode`'s fields, read once the node is in draft-18's names. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Draft18Node { + #[serde(default)] + id: Option, + #[serde(default)] + parent_id: Option, + name: String, + #[serde(with = "time::serde::rfc3339")] + created: UtcDate, + #[serde(default, with = "time::serde::rfc3339::option")] + modified: Option, + #[serde(flatten)] + kind: NodeKind, +} + +impl TryFrom> for FileNode { + type Error = serde_json::Error; + + fn try_from(mut node: Map) -> Result { + to_draft18(&mut node); + let n: Draft18Node = serde_json::from_value(Value::Object(node))?; + Ok(FileNode { + id: n.id, + parent_id: n.parent_id, + name: n.name, + created: n.created, + modified: n.modified, + kind: n.kind, + }) + } +} + +impl NodeKind { + pub fn db_name(&self) -> &'static str { + match self { + NodeKind::File(_) => "file", + NodeKind::Directory(_) => "directory", + NodeKind::SymLink(_) => "symlink", } } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum NodeType { - File, - Directory, - Symlink, +fn bool_or_false<'de, D>(d: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(Option::::deserialize(d)?.unwrap_or_default()) } fn default_true() -> bool { true } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn node(v: Value) -> FileNode { + serde_json::from_value(v).unwrap() + } + + #[test] + fn both_shapes_read_the_same() { + let created = "2022-02-02T02:02:02Z"; + let draft = node( + json!({"id": "F1", "@type": "File", "name": "a.sh", "blobId": "B1", + "mediaType": "text/x-sh", "executable": true, "created": created}), + ); + let legacy = node( + json!({"id": "F1", "nodeType": "file", "name": "a.sh", "blobId": "B1", + "type": "text/x-sh", "created": created}), + ); + for n in [&draft, &legacy] { + let NodeKind::File(f) = &n.kind else { + panic!("{n:?}") + }; + assert_eq!(f.media_type.as_deref(), Some("text/x-sh")); + assert_eq!(f.blob_id.as_ref().map(|b| b.0.as_str()), Some("B1")); + } + let link = node(json!({"id": "L", "nodeType": "symlink", "name": "l", + "target": ["a"], "created": created})); + assert!(matches!(link.kind, NodeKind::SymLink(_))); + let dir = node(json!({"id": "D", "name": "d", "created": created})); + assert!(matches!(dir.kind, NodeKind::Directory(_))); + assert!( + serde_json::from_value::(json!({"id": "S", "nodeType": "socket", + "name": "s", "created": created})) + .is_err() + ); + } + + #[test] + fn a_legacy_server_gets_its_own_names() { + let mut m = json!({"@type": "File", "name": "a.sh", "blobId": "B1", + "mediaType": "text/x-sh", "executable": true}); + let Value::Object(map) = &mut m else { + unreachable!() + }; + let mut draft = map.clone(); + for_server(&mut draft, Shape::Draft18); + assert_eq!(&draft, map); + for_server(map, Shape::Legacy); + assert_eq!( + m, + json!({"nodeType": "file", "name": "a.sh", "blobId": "B1", + "type": "text/x-sh"}) + ); + } +} diff --git a/src/jmap/wire/mod.rs b/src/jmap/wire/mod.rs index 2250540..d034364 100644 --- a/src/jmap/wire/mod.rs +++ b/src/jmap/wire/mod.rs @@ -16,5 +16,6 @@ pub mod identity; pub mod mailbox; pub mod participant_identity; pub mod sieve_script; +pub mod vacation_response; pub use common::{Date, EmailAddress, JmapId, UtcDate}; diff --git a/src/jmap/wire/vacation_response.rs b/src/jmap/wire/vacation_response.rs new file mode 100644 index 0000000..f2e0d73 --- /dev/null +++ b/src/jmap/wire/vacation_response.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +pub const TYPE_NAME: &str = "VacationResponse"; + +pub const SINGLETON_ID: &str = "singleton"; + +pub const RESERVED_SCRIPT_NAME: &str = "vacation"; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VacationResponse { + #[serde(default)] + pub is_enabled: bool, + + #[serde(default)] + pub from_date: Option, + + #[serde(default)] + pub to_date: Option, + + #[serde(default)] + pub subject: Option, + + #[serde(default)] + pub text_body: Option, + + #[serde(default)] + pub html_body: Option, +} + +impl VacationResponse { + pub fn is_unset(&self) -> bool { + !self.is_enabled + && self.from_date.is_none() + && self.to_date.is_none() + && self.subject.is_none() + && self.text_body.is_none() + && self.html_body.is_none() + } + + pub fn is_reserved_script_name(name: &str) -> bool { + name.eq_ignore_ascii_case(RESERVED_SCRIPT_NAME) + } +} diff --git a/src/secret.rs b/src/secret.rs index cabce00..32fcb93 100644 --- a/src/secret.rs +++ b/src/secret.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -29,12 +30,8 @@ pub fn resolve_any( if let Some(value) = flag_value { return Ok(value.to_owned()); } - for env_var in env_vars { - if let Ok(value) = std::env::var(env_var) - && !value.is_empty() - { - return Ok(value); - } + if let Some(value) = from_env(env_vars) { + return Ok(value); } if std::io::stdin().is_terminal() { return prompt_no_echo(prompt_label); @@ -45,6 +42,13 @@ pub fn resolve_any( ))) } +/// The first of `env_vars` that is set and not empty. +pub fn from_env(env_vars: &[&str]) -> Option { + env_vars + .iter() + .find_map(|var| std::env::var(var).ok().filter(|v| !v.is_empty())) +} + fn prompt_no_echo(label: &str) -> Result { let mut stderr = std::io::stderr(); write!(stderr, "{label}: ")?; diff --git a/src/sync/export.rs b/src/sync/export.rs index 756fb85..9500d49 100644 --- a/src/sync/export.rs +++ b/src/sync/export.rs @@ -20,22 +20,24 @@ use crate::jmap::blobxfer; use crate::jmap::connect::{self, Connected}; use crate::jmap::error::JmapError; use crate::jmap::http::HttpClient; -use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_ids, set_call}; +use crate::jmap::request::{ + Request, SetRequest, URN_VACATION_RESPONSE, get_all, get_objects, query_all_ids, set_call, +}; use crate::jmap::session::{Limits, Session}; use crate::jmap::wire::JmapId; +use crate::jmap::wire::vacation_response; use crate::logging::{LEVEL_DEFAULT, Logger}; use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver}; use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts}; use crate::types::ObjectType; -const EXPORT_ORDER: [ObjectType; 10] = [ +const EXPORT_ORDER: [ObjectType; 9] = [ ObjectType::Mailbox, ObjectType::AddressBook, ObjectType::Calendar, ObjectType::FileNode, ObjectType::Identity, ObjectType::SieveScript, - ObjectType::ParticipantIdentity, ObjectType::Email, ObjectType::ContactCard, ObjectType::CalendarEvent, @@ -417,6 +419,35 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result } } + if config + .objects + .as_ref() + .is_none_or(|s| s.contains(&ObjectType::SieveScript)) + && net.session.may_support(&net.account, URN_VACATION_RESPONSE) + { + let mut counts = TypeCounts::default(); + let present = match vacation::reconcile(&ctx, &net, &mut counts, &logger) { + Ok(present) => present, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!( + "type {} aborted: {e}", + vacation_response::TYPE_NAME + )); + counts.failed += 1; + true + } + }; + if present { + if ctx.dry_run() { + dry_rows.push((vacation_response::TYPE_NAME, counts.updated, 0, 0)); + } + summary + .per_type + .push((vacation_response::TYPE_NAME, counts)); + } + } + if ctx.dry_run() { let would_fail = net.would_fail.lock().expect("would-fail list").clone(); print_plan(&summary, &dry_rows, &would_fail, config.prune); @@ -487,6 +518,16 @@ fn work_list( logger: &Logger, ) -> Vec { let selected = config.objects.as_ref(); + let participant = ObjectType::ParticipantIdentity; + if selected.is_none_or(|s| s.contains(&participant)) + && has_rows(conn, participant) + && logger.enabled(LEVEL_DEFAULT) + { + eprintln!( + "export: {} not exported; the target creates participant identities from the account's addresses", + participant.jmap_name() + ); + } EXPORT_ORDER .into_iter() .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true)) @@ -523,9 +564,7 @@ fn reconcile_type( flat::reconcile(ctx, net, ty, maps, counts, logger) } ObjectType::Identity => keyed::reconcile_identity(ctx, net, maps, counts, logger), - ObjectType::ParticipantIdentity => { - keyed::reconcile_participant(ctx, net, maps, counts, logger) - } + ObjectType::ParticipantIdentity => Ok(Plan::default()), ObjectType::SieveScript => sieve::reconcile(ctx, net, maps, counts, logger), ObjectType::ContactCard | ObjectType::CalendarEvent => { uidtype::reconcile(ctx, net, ty, maps, counts, logger) @@ -552,7 +591,7 @@ pub struct Plan { } fn do_destroy(net: &Net, ty: ObjectType, plan: &Plan, logger: &Logger, counts: &mut TypeCounts) { - if ty == ObjectType::SieveScript { + if ty == ObjectType::SieveScript && plan.active_sieve_target.is_none() { deactivate_active_sieve_script(net, logger); } let destroy = Value::Array( @@ -700,6 +739,7 @@ mod keyed; mod sieve; mod sieve_names; +mod vacation; mod uidtype; diff --git a/src/sync/export/flat.rs b/src/sync/export/flat.rs index 06f0070..e8ef899 100644 --- a/src/sync/export/flat.rs +++ b/src/sync/export/flat.rs @@ -12,7 +12,7 @@ use super::common::{create_batch, jid, target_get_all}; use super::{Maps, Net, Plan}; use crate::error::Error; use crate::jmap::wire::JmapId; -use crate::logging::Logger; +use crate::logging::{LEVEL_PROGRESS, Logger}; use crate::sync::import_jmap::mapping::{ ADDRESS_BOOK_SELECT, CALENDAR_SELECT, row_to_address_book, row_to_calendar, }; @@ -83,7 +83,16 @@ pub fn reconcile( Ok(if ty == ObjectType::AddressBook { row_to_address_book(row).map(|w| serde_json::to_value(&w)) } else { - row_to_calendar(row).map(|w| serde_json::to_value(&w)) + row_to_calendar(row).map(|mut w| { + if let Some(dropped) = w.normalize_color() + && logger.enabled(LEVEL_PROGRESS) + { + eprintln!( + "Calendar {local}: dropping color {dropped:?}, not a CSS color name or #RGB/#RRGGBB value" + ); + } + serde_json::to_value(&w) + }) }) }) .map_err(|e| Error::Partial(e.to_string()))? diff --git a/src/sync/export/keyed.rs b/src/sync/export/keyed.rs index c1e565f..3d45853 100644 --- a/src/sync/export/keyed.rs +++ b/src/sync/export/keyed.rs @@ -12,10 +12,8 @@ use super::common::{create_batch, target_get_all}; use super::{Maps, Net, Plan}; use crate::error::Error; use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{ - IDENTITY_SELECT, PARTICIPANT_IDENTITY_SELECT, row_to_identity, row_to_participant_identity, -}; -use crate::sync::keys::{identity_key, participant_identity_key}; +use crate::sync::import_jmap::mapping::{IDENTITY_SELECT, row_to_identity}; +use crate::sync::keys::identity_key; use crate::sync::{Context, TypeCounts}; use crate::types::ObjectType; @@ -99,47 +97,3 @@ pub fn reconcile_identity( rows, ) } - -pub fn reconcile_participant( - ctx: &Context, - net: &Net, - _maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let mut stmt = ctx - .conn - .prepare(PARTICIPANT_IDENTITY_SELECT) - .map_err(|e| Error::Partial(e.to_string()))?; - let rows: Vec<(i64, [u8; 32], Value)> = stmt - .query_map([], |row| { - let id: i64 = row.get(0)?; - Ok((id, row_to_participant_identity(row))) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - .into_iter() - .map(|(id, w)| { - let w = w.map_err(Error::from)?; - let key = participant_identity_key(&w.calendar_address, &w.name); - let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?; - Ok((id, key, v)) - }) - .collect::>()?; - drop(stmt); - run( - net, - ObjectType::ParticipantIdentity, - counts, - logger, - |t| { - Some(participant_identity_key( - t.get("calendarAddress") - .and_then(Value::as_str) - .unwrap_or(""), - t.get("name").and_then(Value::as_str).unwrap_or(""), - )) - }, - rows, - ) -} diff --git a/src/sync/export/sieve.rs b/src/sync/export/sieve.rs index 11f560d..a5343cd 100644 --- a/src/sync/export/sieve.rs +++ b/src/sync/export/sieve.rs @@ -12,10 +12,12 @@ use serde_json::{Value, json}; use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all, update_batch}; use super::sieve_names; use super::{Maps, Net, Plan, Uploader}; +use crate::db; use crate::error::Error; use crate::jmap::blobxfer; use crate::jmap::error::JmapError; -use crate::jmap::request::{Request, URN_INBUXA, check_method_error}; +use crate::jmap::request::{Request, URN_INBUXA, URN_VACATION_RESPONSE, check_method_error}; +use crate::jmap::wire::vacation_response::VacationResponse; use crate::logging::{LEVEL_DEFAULT, Logger}; use crate::sync::import_jmap::mapping::BlobBytes; use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script}; @@ -44,7 +46,11 @@ pub fn reconcile( target_by_name.insert(name.to_owned(), id); } - let locals: Vec<(i64, Option, bool, i64)> = { + let target_has_vacation = net.session.may_support(&net.account, URN_VACATION_RESPONSE); + let vacation_exported = target_has_vacation + && db::vacation_response::exists(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))?; + + let mut locals: Vec<(i64, Option, bool, i64)> = { let mut stmt = ctx .conn .prepare(SIEVE_SELECT) @@ -63,6 +69,16 @@ pub fn reconcile( .collect::>()? }; + if vacation_exported { + let before = locals.len(); + locals.retain(|(_, name, _, _)| { + !name + .as_deref() + .is_some_and(VacationResponse::is_reserved_script_name) + }); + counts.skipped += (before - locals.len()) as u64; + } + let mut active_target: Option = None; let mut deactivate = false; let mut uploader = Uploader::new(net, &ctx.conn); @@ -222,6 +238,9 @@ pub fn reconcile( let mut prune_candidates: Vec = target_by_name .iter() .filter(|(name, _)| !local_names.contains(*name)) + .filter(|(name, _)| { + !(target_has_vacation && VacationResponse::is_reserved_script_name(name)) + }) .map(|(_, id)| id.clone()) .collect(); prune_candidates.sort(); diff --git a/src/sync/export/tree.rs b/src/sync/export/tree.rs index 188f55e..dd1c7be 100644 --- a/src/sync/export/tree.rs +++ b/src/sync/export/tree.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -7,12 +8,14 @@ use std::collections::{HashMap, HashSet}; use rusqlite::params; -use serde_json::Value; +use serde_json::{Value, json}; use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get}; use super::{Net, Plan, Uploader}; use crate::error::Error; +use crate::jmap::request::{Request, URN_INBUXA}; use crate::jmap::wire::JmapId; +use crate::jmap::wire::file_node::{NodeKind, Shape, for_server}; use crate::logging::Logger; use crate::sync::import_jmap::mapping::{ FILE_NODE_SELECT, MAILBOX_SELECT, TargetResolver, row_to_file_node, row_to_mailbox, @@ -61,24 +64,72 @@ fn load_local(ctx: &Context, ty: ObjectType) -> Result, Error> { fn load_target(net: &Net, ty: ObjectType) -> Result, Error> { let props: &[&str] = &["role", "name", "parentId", "myRights"]; let list = target_query_get(net, ty, Some(props)).map_err(Error::from)?; + let mut own_rights: HashMap<&str, bool> = HashMap::with_capacity(list.len()); + for v in &list { + if let (Some(id), Some(may)) = ( + v.get("id").and_then(Value::as_str), + v.get("myRights") + .and_then(|r| r.get("mayDelete")) + .and_then(Value::as_bool), + ) { + own_rights.insert(id, may); + } + } Ok(list .iter() .filter_map(|v| { + let id = jid(v)?; + let parent = v.get("parentId").and_then(Value::as_str); + let may_delete = own_rights + .get(id.as_str()) + .or_else(|| parent.and_then(|p| own_rights.get(p))) + .copied() + .unwrap_or(true); Some(TargetNode { - id: jid(v)?, - parent: v.get("parentId").and_then(Value::as_str).map(str::to_owned), + id, + parent: parent.map(str::to_owned), name: v.get("name").and_then(Value::as_str)?.to_owned(), role: v.get("role").and_then(Value::as_str).map(str::to_owned), - may_delete: v - .get("myRights") - .and_then(|r| r.get("mayDelete")) - .and_then(Value::as_bool) - .unwrap_or(true), + may_delete, }) }) .collect()) } +/// The FileNode shape the target takes. Nothing in the session says, so this +/// asks with a query sorted by `type`: a server on the 0.16 line answers it, +/// and a draft-18 one refuses it as `unsupportedSort`, the property being +/// gone. It writes nothing. Any other answer leaves it to the session, and +/// inbuxa takes the 0.16 shape. +fn file_node_shape(net: &Net) -> Shape { + let mut req = Request::new(); + req.call( + "FileNode/query", + json!({ + "accountId": net.account, + "sort": [{"property": "type", "isAscending": true}], + "limit": 1, + }), + "shape", + ); + let answer = req.send(&net.client, &net.api); + let call = answer + .as_ref() + .ok() + .and_then(|r| r.by_call_id("shape").ok()); + match call { + Some(c) if c.name == "FileNode/query" => Shape::Legacy, + Some(c) + if c.name == "error" + && c.args.get("type").and_then(Value::as_str) == Some("unsupportedSort") => + { + Shape::Draft18 + } + _ if net.session.capabilities.contains_key(URN_INBUXA) => Shape::Legacy, + _ => Shape::Draft18, + } +} + fn depth(local: i64, by: &HashMap>) -> usize { let mut d = 0; let mut cur = by.get(&local).copied().flatten(); @@ -197,6 +248,7 @@ pub fn reconcile( .max() .unwrap_or(0); + let shape = (ty == ObjectType::FileNode && !to_create.is_empty()).then(|| file_node_shape(net)); let mut uploader = Uploader::new(net, &ctx.conn); let mut taken_roles: HashSet = targets.iter().filter_map(|t| t.role.clone()).collect(); let interleave = ty == ObjectType::FileNode; @@ -240,7 +292,7 @@ pub fn reconcile( } let cid = format!("c{}", n.local); let _ = uploader.take_touched(); - let obj = match build_create(ctx, ty, n.local, maps, &mut uploader) { + let obj = match build_create(ctx, ty, n.local, maps, &mut uploader, shape) { Ok(o) => o, Err(e) => { logger.warn(&format!( @@ -262,7 +314,7 @@ pub fn reconcile( &mut uploader, touched, outcome, - |up| build_create(ctx, ty, n.local, maps, up), + |up| build_create(ctx, ty, n.local, maps, up, shape), ) { Ok(o) => o, Err(e) => { @@ -335,7 +387,7 @@ pub fn reconcile( counts.skipped += 1; continue; } - match build_create(ctx, ty, n.local, maps, &mut uploader) { + match build_create(ctx, ty, n.local, maps, &mut uploader, shape) { Ok(mut obj) => { if ty == ObjectType::Mailbox && let Some(r) = n.role.as_deref() @@ -418,6 +470,7 @@ fn build_create( local: i64, maps: &Maps, uploader: &mut Uploader<'_>, + shape: Option, ) -> Result { if ty == ObjectType::Mailbox { let wire = ctx @@ -441,21 +494,20 @@ fn build_create( .map_err(|e| Error::Partial(e.to_string()))? .map_err(Error::from)?; let mut wire = fnrow.wire; - if let (crate::jmap::wire::file_node::NodeType::File, Some(blob_local)) = - (wire.effective_node_type(), fnrow.blob_local_id) - { - let ct = wire + if let (NodeKind::File(file), Some(blob_local)) = (&mut wire.kind, fnrow.blob_local_id) { + let ct = file .media_type - .clone() - .unwrap_or_else(|| "application/octet-stream".to_owned()); - let id = uploader.upload_with(blob_local, &ct).map_err(Error::from)?; - wire.blob_id = Some(id); + .as_deref() + .unwrap_or("application/octet-stream"); + let id = uploader.upload_with(blob_local, ct).map_err(Error::from)?; + file.blob_id = Some(id); } let mut obj = serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()))?; if let Value::Object(m) = &mut obj { m.remove("created"); m.remove("modified"); m.remove("isSubscribed"); + for_server(m, shape.unwrap_or(Shape::Draft18)); } Ok(obj) } diff --git a/src/sync/export/vacation.rs b/src/sync/export/vacation.rs new file mode 100644 index 0000000..1c9035c --- /dev/null +++ b/src/sync/export/vacation.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::json; + +use super::Net; +use crate::db; +use crate::error::Error; +use crate::jmap::request::{SetRequest, set_call}; +use crate::jmap::wire::vacation_response::{SINGLETON_ID, TYPE_NAME}; +use crate::logging::Logger; +use crate::sync::{Context, TypeCounts}; + +pub fn reconcile( + ctx: &Context, + net: &Net, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let Some(vacation) = + db::vacation_response::load(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))? + else { + return Ok(false); + }; + if net.dry_run { + counts.updated += 1; + return Ok(true); + } + let outcome = set_call( + &net.client, + &net.api, + &net.account, + TYPE_NAME, + SetRequest { + update: Some(json!({ SINGLETON_ID: vacation })), + ..Default::default() + }, + &net.limits, + ) + .map_err(Error::from)?; + counts.updated += outcome.updated.len() as u64; + for (id, err) in &outcome.not_updated { + logger.warn(&format!("{TYPE_NAME} {id} not updated: {err}")); + counts.failed += 1; + } + Ok(true) +} diff --git a/src/sync/import_dav.rs b/src/sync/import_dav.rs index 0b2af29..81779d1 100644 --- a/src/sync/import_dav.rs +++ b/src/sync/import_dav.rs @@ -8,9 +8,10 @@ pub mod calcard; pub mod collections; pub mod coordinator; pub mod items; +pub mod replace; pub mod tree; -pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run, run_reporting}; +pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, SourceChange, run, run_reporting}; use crate::error::Error; use crate::jmap::error::JmapError; diff --git a/src/sync/import_dav/calcard.rs b/src/sync/import_dav/calcard.rs index 217b294..e6af6ab 100644 --- a/src/sync/import_dav/calcard.rs +++ b/src/sync/import_dav/calcard.rs @@ -5,16 +5,22 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::borrow::Cow; +use std::fmt::Write; + +use calcard::Entry; use calcard::icalendar::ICalendar; use calcard::vcard::VCard; use serde_json::{Map, Value}; +const MAX_INVALID_LINE_CHARS: usize = 80; + #[derive(Debug, thiserror::Error)] pub enum CalcardError { - #[error("vcard parse failed")] - VCardParse, - #[error("icalendar parse failed")] - ICalParse, + #[error("vcard parse failed: {0}")] + VCardParse(String), + #[error("icalendar parse failed: {0}")] + ICalParse(String), #[error("jscontact serialisation: {0}")] Json(#[from] serde_json::Error), #[error("no parseable entries in iCalendar")] @@ -30,7 +36,8 @@ pub struct JsContact { } pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result { - let vcard = VCard::parse(text).map_err(|_| CalcardError::VCardParse)?; + let vcard = VCard::parse(normalize_line_breaks(text)) + .map_err(|e| CalcardError::VCardParse(describe_parse_error(e)))?; let js = vcard.into_jscontact::(); let value: Value = serde_json::from_str(&js.to_string_pretty())?; let uid = value @@ -82,7 +89,8 @@ impl EntryType { } pub fn ical_to_jscalendar_entries(text: &str) -> Result, CalcardError> { - let ical = ICalendar::parse(text).map_err(|_| CalcardError::ICalParse)?; + let ical = ICalendar::parse(normalize_line_breaks(text)) + .map_err(|e| CalcardError::ICalParse(describe_parse_error(e)))?; let js = ical.into_jscalendar::(); let value: Value = serde_json::from_str(&js.to_string_pretty())?; let mut out = Vec::new(); @@ -93,6 +101,67 @@ pub fn ical_to_jscalendar_entries(text: &str) -> Result, Ca Ok(out) } +fn normalize_line_breaks(text: &str) -> Cow<'_, str> { + let mut after_cr = text.split('\r').skip(1).peekable(); + let mut has_bare_cr = false; + while let Some(rest) = after_cr.next() { + if is_bare_cr(rest, after_cr.peek().is_some()) { + has_bare_cr = true; + break; + } + } + if !has_bare_cr { + return Cow::Borrowed(text); + } + let mut parts = text.split('\r').peekable(); + let mut out = String::with_capacity(text.len()); + out.push_str(parts.next().unwrap_or_default()); + while let Some(rest) = parts.next() { + if is_bare_cr(rest, parts.peek().is_some()) { + out.push('\n'); + } + out.push_str(rest); + } + Cow::Owned(out) +} + +fn is_bare_cr(rest: &str, more: bool) -> bool { + let followed_by_cr = rest.is_empty() && more; + !followed_by_cr && !rest.starts_with('\n') +} + +fn describe_parse_error(entry: Entry) -> String { + let mut out = String::new(); + match entry { + Entry::InvalidLine(line) => { + out.push_str("invalid line "); + let mut chars = line.chars(); + let head: String = chars.by_ref().take(MAX_INVALID_LINE_CHARS).collect(); + let _ = write!(out, "{head:?}"); + if chars.next().is_some() { + out.push_str(" (truncated)"); + } + } + Entry::UnexpectedComponentEnd { expected, found } => { + let _ = write!( + out, + "found END:{} where END:{} was expected", + found.as_str(), + expected.as_str() + ); + } + Entry::UnterminatedComponent(name) => { + let _ = write!(out, "{name} is never closed"); + } + Entry::TooManyComponents => out.push_str("too many components"), + Entry::Eof => out.push_str("no BEGIN line found"), + Entry::VCard(_) => out.push_str("found a vCard where an iCalendar was expected"), + Entry::ICalendar(_) => out.push_str("found an iCalendar where a vCard was expected"), + _ => out.push_str("unrecognised content"), + } + out +} + fn collect_entries(value: Value, out: &mut Vec) { match value { Value::Object(map) => { @@ -228,7 +297,53 @@ END:VCALENDAR\r\n"; #[test] fn vcard_garbage_fails_parse() { let err = vcard_to_jscontact("not a vcard", "/x").unwrap_err(); - assert!(matches!(err, CalcardError::VCardParse)); + assert!(matches!(err, CalcardError::VCardParse(_))); + } + + // Kerio Connect sends calendar-data with bare CR line breaks (issue #48). + #[test] + fn ical_with_bare_cr_line_breaks_parses() { + let ical = SAMPLE_ICAL.replace("\r\n", "\r"); + let entries = ical_to_jscalendar_entries(&ical).expect("bare CR parses"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].data["title"], "Test Event"); + } + + // A folded line split by a bare CR still unfolds. + #[test] + fn ical_with_bare_cr_unfolds_continuation_lines() { + let ical = "BEGIN:VCALENDAR\rVERSION:2.0\rBEGIN:VEVENT\rUID:k1@example\r\ + DTSTAMP:20260101T000000Z\rDTSTART:20260101T090000Z\r\ + SUMMARY:Quarterly\r review\rEND:VEVENT\rEND:VCALENDAR\r"; + let entries = ical_to_jscalendar_entries(ical).expect("parse"); + assert_eq!(entries[0].data["title"], "Quarterly review"); + } + + #[test] + fn vcard_with_bare_cr_line_breaks_parses() { + let vcard = SAMPLE_VCARD.replace("\r\n", "\r"); + let jc = vcard_to_jscontact(&vcard, "/x").expect("bare CR parses"); + assert_eq!(jc.uid, "abc-123"); + } + + #[test] + fn line_break_normalization_borrows_unless_bare_cr() { + for text in ["A\nB\n", "A\r\nB\r\n", "A\r\r\nB\r\r\n", "no breaks"] { + assert!( + matches!(normalize_line_breaks(text), Cow::Borrowed(_)), + "{text:?}" + ); + } + assert_eq!(normalize_line_breaks("A\rB\r\nC\r"), "A\nB\nC\n"); + } + + #[test] + fn ical_parse_error_names_the_offending_line() { + let err = ical_to_jscalendar_entries("Content-Type: text/calendar\r\nBEGIN:VCALENDAR\r\n") + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("invalid line"), "{msg}"); + assert!(msg.contains("Content-Type"), "{msg}"); } #[test] diff --git a/src/sync/import_dav/coordinator.rs b/src/sync/import_dav/coordinator.rs index 42c0766..ad2e67a 100644 --- a/src/sync/import_dav/coordinator.rs +++ b/src/sync/import_dav/coordinator.rs @@ -18,6 +18,7 @@ use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; use super::collections; use super::items; +use super::replace::{self, Removed}; use super::tree; use crate::net::CertOverride; @@ -73,6 +74,13 @@ impl DavAuth { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceChange { + Reject, + Allow, + Replace, +} + #[derive(Debug)] pub struct DavImportConfig { pub kind: DavKindArg, @@ -81,7 +89,7 @@ pub struct DavImportConfig { pub allow_cleartext: bool, pub dav_connections: usize, pub multiget_batch: usize, - pub allow_source_change: bool, + pub source_change: SourceChange, } pub fn run(common: CommonConfig, config: DavImportConfig) -> Result { @@ -128,6 +136,7 @@ fn run_into( }; if !common.dry_run + && config.source_change == SourceChange::Reject && let Some((url, acc)) = db::sources::conflicting_source( &conn, config.kind.source_kind(), @@ -135,11 +144,11 @@ fn run_into( &account_id, ) .map_err(|e| Error::Partial(e.to_string()))? - && !config.allow_source_change { return Err(Error::SourceChange(format!( "archive already records {} source ({url}, account {acc}); \ - pass --allow-source-change to import a different account", + pass --allow-source-change to import a different account, \ + or --replace-source to drop the recorded one", config.kind.source_kind() ))); } @@ -154,12 +163,27 @@ fn run_into( } if common.dry_run { + if config.source_change == SourceChange::Replace && logger.enabled(LEVEL_DEFAULT) { + for (_, url, acc) in + db::sources::other_sources(&conn, &key.kind, &session_url, &account_id)? + { + eprintln!( + "dry-run: would remove {} source {url} (account {acc})", + key.kind + ); + } + } let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary); summary.retries_observed = client.retries_observed(); summary.retry_after_sleeps = client.retry_after_sleeps(); return result; } + let removed = match config.source_change { + SourceChange::Replace => replace::remove_other_sources(&conn, &key, logger)?, + SourceChange::Reject | SourceChange::Allow => Removed::default(), + }; + let username = config.auth.username(); let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username) .map_err(|e| Error::Partial(e.to_string()))?; @@ -177,6 +201,7 @@ fn run_into( }; *summary = phase.summary; + removed.add_to(summary); summary.retries_observed = client.retries_observed(); summary.retry_after_sleeps = client.retry_after_sleeps(); if let Some(e) = phase.error { diff --git a/src/sync/import_dav/replace.rs b/src/sync/import_dav/replace.rs new file mode 100644 index 0000000..4456ee2 --- /dev/null +++ b/src/sync/import_dav/replace.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, params}; + +use crate::db::dav_ids::{ADDRESS_BOOK, CALENDAR, CALENDAR_EVENT, CONTACT_CARD, FILE_NODE}; +use crate::db::sources::{self, SourceKey}; +use crate::error::Error; +use crate::logging::{LEVEL_DEFAULT, Logger}; +use crate::sync::Summary; + +const DELETE_MAPPED: [(&str, &str); 4] = [ + ( + CALENDAR_EVENT, + "DELETE FROM calendar_events WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), + ( + CONTACT_CARD, + "DELETE FROM contact_cards WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), + ( + CALENDAR, + "DELETE FROM calendars WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), + ( + ADDRESS_BOOK, + "DELETE FROM address_books WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), +]; + +const DELETE_LEAF_FILE_NODES: &str = "DELETE FROM file_nodes WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2) + AND NOT EXISTS (SELECT 1 FROM file_nodes AS child WHERE child.parent_id = file_nodes.id)"; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Removed { + pub calendar_events: u64, + pub contact_cards: u64, + pub calendars: u64, + pub address_books: u64, + pub file_nodes: u64, +} + +impl Removed { + fn slot(&mut self, type_name: &str) -> Option<&mut u64> { + match type_name { + CALENDAR_EVENT => Some(&mut self.calendar_events), + CONTACT_CARD => Some(&mut self.contact_cards), + CALENDAR => Some(&mut self.calendars), + ADDRESS_BOOK => Some(&mut self.address_books), + FILE_NODE => Some(&mut self.file_nodes), + _ => None, + } + } + + pub fn add_to(mut self, summary: &mut Summary) { + for (label, counts) in &mut summary.per_type { + if let Some(n) = self.slot(label) { + counts.deleted += std::mem::take(n); + } + } + } +} + +pub fn remove_other_sources( + conn: &Connection, + key: &SourceKey, + logger: Logger, +) -> Result { + let others = sources::other_sources(conn, &key.kind, &key.session_url, &key.account_id)?; + let mut removed = Removed::default(); + if others.is_empty() { + return Ok(removed); + } + let tx = conn.unchecked_transaction()?; + for (source_id, session_url, account_id) in &others { + remove_source(&tx, *source_id, &mut removed)?; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "import: removed {} source {session_url} (account {account_id})", + key.kind + ); + } + } + tx.commit()?; + Ok(removed) +} + +fn remove_source( + conn: &Connection, + source_id: i64, + removed: &mut Removed, +) -> Result<(), rusqlite::Error> { + for (type_name, sql) in DELETE_MAPPED { + let n = conn.execute(sql, params![source_id, type_name])?; + if let Some(slot) = removed.slot(type_name) { + *slot += n as u64; + } + } + let mut leaves = conn.prepare(DELETE_LEAF_FILE_NODES)?; + loop { + let n = leaves.execute(params![source_id, FILE_NODE])?; + if n == 0 { + break; + } + removed.file_nodes += n as u64; + } + sources::delete_source(conn, source_id) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{dav_ids, init}; + use crate::sync::TypeCounts; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c.pragma_update(None, "foreign_keys", "ON").unwrap(); + c + } + + fn source(c: &Connection, kind: &str, url: &str) -> (SourceKey, i64) { + let key = SourceKey { + kind: kind.to_owned(), + session_url: url.to_owned(), + account_id: format!("{url}/u/"), + }; + let id = sources::upsert_source(c, &key, None, "u").unwrap(); + (key, id) + } + + fn file_node(c: &Connection, sid: i64, parent: Option, name: &str, href: &str) -> i64 { + c.execute( + "INSERT INTO file_nodes (parent_id, node_type, name, created) + VALUES (?1, 'directory', ?2, '2026-01-01T00:00:00Z')", + params![parent, name], + ) + .unwrap(); + let id = c.last_insert_rowid(); + dav_ids::insert(c, sid, FILE_NODE, href, href, "", id).unwrap(); + id + } + + fn count(c: &Connection, table: &str) -> i64 { + c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + #[test] + fn replacing_a_webdav_source_drops_the_old_tree_leaf_first() { + let c = mem(); + let (_, old) = source(&c, "webdav", "https://old"); + let a = file_node(&c, old, None, "a", "/a/"); + let b = file_node(&c, old, Some(a), "b", "/a/b/"); + file_node(&c, old, Some(b), "c", "/a/b/c/"); + let (new_key, new) = source(&c, "webdav", "https://new"); + file_node(&c, new, None, "a", "/a/"); + + let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap(); + + assert_eq!(removed.file_nodes, 3); + assert_eq!(count(&c, "file_nodes"), 1); + assert_eq!(count(&c, "sources"), 1); + assert_eq!(count(&c, "sync_id_dav"), 1); + let kept: i64 = c + .query_row("SELECT source_id FROM sync_id_dav", [], |r| r.get(0)) + .unwrap(); + assert_eq!(kept, new); + } + + #[test] + fn replacing_leaves_other_kinds_untouched() { + let c = mem(); + let (_, cal) = source(&c, "caldav", "https://old"); + c.execute("INSERT INTO calendars (name) VALUES ('Work')", []) + .unwrap(); + let cal_local = c.last_insert_rowid(); + dav_ids::insert(&c, cal, CALENDAR, "/c/", "/c/", "", cal_local).unwrap(); + c.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')", + params![format!("[{cal_local}]")], + ) + .unwrap(); + let ev = c.last_insert_rowid(); + dav_ids::insert(&c, cal, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap(); + let (_, old_web) = source(&c, "webdav", "https://old"); + file_node(&c, old_web, None, "a", "/a/"); + let (new_web, _) = source(&c, "webdav", "https://new"); + + let removed = remove_other_sources(&c, &new_web, Logger::from_flags(true, 0)).unwrap(); + + assert_eq!(removed.file_nodes, 1); + assert_eq!(removed.calendars + removed.calendar_events, 0); + assert_eq!(count(&c, "calendars"), 1); + assert_eq!(count(&c, "calendar_events"), 1); + assert_eq!(count(&c, "file_nodes"), 0); + } + + #[test] + fn replacing_a_caldav_source_drops_its_calendars_and_events() { + let c = mem(); + let (_, old) = source(&c, "caldav", "https://old"); + c.execute("INSERT INTO calendars (name) VALUES ('Work')", []) + .unwrap(); + let cal_local = c.last_insert_rowid(); + dav_ids::insert(&c, old, CALENDAR, "/c/", "/c/", "", cal_local).unwrap(); + c.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')", + params![format!("[{cal_local}]")], + ) + .unwrap(); + let ev = c.last_insert_rowid(); + dav_ids::insert(&c, old, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap(); + let (new_key, _) = source(&c, "caldav", "https://new"); + + let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap(); + + assert_eq!((removed.calendars, removed.calendar_events), (1, 1)); + assert_eq!(count(&c, "calendars"), 0); + assert_eq!(count(&c, "calendar_events"), 0); + assert_eq!(count(&c, "sync_id_dav"), 0); + } + + #[test] + fn no_other_source_is_a_no_op() { + let c = mem(); + let (key, _) = source(&c, "webdav", "https://only"); + let removed = remove_other_sources(&c, &key, Logger::from_flags(true, 0)).unwrap(); + assert_eq!(removed, Removed::default()); + assert_eq!(count(&c, "sources"), 1); + } + + #[test] + fn removed_counts_land_on_matching_summary_rows() { + let mut summary = Summary::default(); + summary.per_type.push((FILE_NODE, TypeCounts::default())); + Removed { + file_nodes: 4, + calendars: 9, + ..Removed::default() + } + .add_to(&mut summary); + assert_eq!(summary.per_type.len(), 1); + assert_eq!(summary.per_type[0].1.deleted, 4); + } +} diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs index 13d24f0..e276c87 100644 --- a/src/sync/import_imap/coordinator.rs +++ b/src/sync/import_imap/coordinator.rs @@ -395,7 +395,7 @@ fn run_into( let local_mailboxes = db::imap_ids::mailbox_folders(&conn, source_id)?; let server_folder_set: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); - upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?; + upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts, logger)?; let server_delimiter = resolved.iter().find_map(|f| f.delimiter).unwrap_or('/'); @@ -436,7 +436,7 @@ fn run_into( ) .map_err(|e| Error::Connection(format!("worker pool: {e}")))?; - for (i, folder) in resolved.iter().enumerate() { + for (i, folder) in resolved.iter().filter(|f| f.selectable).enumerate() { if i > 0 { let _ = control_run_collect(&mut client, &control_ctx, "NOOP"); } @@ -614,16 +614,29 @@ fn upsert_mailboxes( source_id: i64, folders: &[ResolvedFolder], counts: &mut TypeCounts, + logger: Logger, ) -> Result<(), Error> { let tx = conn.transaction()?; let mut local_ids: HashMap = HashMap::new(); for folder in folders { let parent_local = match &folder.parent_path { - Some(p) => local_ids.get(p.as_str()).copied().or_else(|| { - db::imap_ids::local_for_mailbox(&tx, source_id, p) - .ok() - .flatten() - }), + Some(p) => { + let found = match local_ids.get(p.as_str()) { + Some(id) => Some(*id), + None => db::imap_ids::local_for_mailbox(&tx, source_id, p)?, + }; + if found.is_none() { + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "folder {:?}: parent {p:?} could not be resolved; storing it at the top level", + folder.name + ), + ); + } + found + } None => None, }; let existing = db::imap_ids::local_for_mailbox(&tx, source_id, &folder.name)?; @@ -1290,7 +1303,7 @@ fn dry_run_summary( ); } - for folder in folders { + for folder in folders.iter().filter(|f| f.selectable) { let select_resp = match select_folder(client, control_ctx, folder) { Ok(r) => r, Err(e) => { diff --git a/src/sync/import_imap/folders.rs b/src/sync/import_imap/folders.rs index cea6a18..9020a41 100644 --- a/src/sync/import_imap/folders.rs +++ b/src/sync/import_imap/folders.rs @@ -39,6 +39,7 @@ pub struct ResolvedFolder { pub delimiter: Option, pub role: Option<&'static str>, pub subscribed: bool, + pub selectable: bool, pub status: Option, } @@ -131,16 +132,16 @@ pub fn apply_filters( folders: Vec, filters: &FolderFilters, ) -> Vec { - let mut keep: Vec = folders - .into_iter() - .filter(|f| f.selectable) - .filter(|f| match_filters(&f.name, filters)) - .collect(); - if filters.subscribed_only { - keep.retain(|f| f.subscribed); - } - let mut resolved: Vec = Vec::with_capacity(keep.len()); - for f in keep { + let mut resolved: Vec = Vec::with_capacity(folders.len()); + let mut others: HashMap = HashMap::new(); + for f in folders { + if !f.selectable + || !match_filters(&f.name, filters) + || (filters.subscribed_only && !f.subscribed) + { + others.insert(f.name.clone(), f); + continue; + } let role = automap::role_for_folder( &f.name, &f.attributes, @@ -153,21 +154,77 @@ pub fn apply_filters( .iter() .any(|s| s.eq_ignore_ascii_case(r)) { + others.insert(f.name.clone(), f); continue; } - let delim = f.delimiter; - let (leaf, parent_path) = split_parent(&f.name, delim); + let (leaf, parent_path) = split_parent(&f.name, f.delimiter); resolved.push(ResolvedFolder { name: f.name, wire_name: f.wire_name, leaf, parent_path, - delimiter: delim, + delimiter: f.delimiter, role, subscribed: f.subscribed, + selectable: true, status: f.status, }); } + let mut present: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); + let mut structural: Vec = Vec::new(); + for child in &resolved { + let mut parent = child.parent_path.clone(); + let mut child_wire = child.wire_name.clone(); + let mut delim = child.delimiter; + while let Some(name) = parent { + if present.contains(&name) { + break; + } + let ancestor = match others.remove(&name) { + Some(d) => { + let delimiter = d.delimiter.or(delim); + let (leaf, parent_path) = split_parent(&d.name, delimiter); + ResolvedFolder { + name: d.name, + wire_name: d.wire_name, + leaf, + parent_path, + delimiter, + role: None, + subscribed: d.subscribed, + selectable: false, + status: None, + } + } + None => { + let wire_name = match delim { + Some(d) => child_wire + .rsplit_once(d) + .map_or_else(|| name.clone(), |(p, _)| p.to_owned()), + None => name.clone(), + }; + let (leaf, parent_path) = split_parent(&name, delim); + ResolvedFolder { + name, + wire_name, + leaf, + parent_path, + delimiter: delim, + role: None, + subscribed: false, + selectable: false, + status: None, + } + } + }; + present.insert(ancestor.name.clone()); + parent = ancestor.parent_path.clone(); + child_wire.clone_from(&ancestor.wire_name); + delim = ancestor.delimiter; + structural.push(ancestor); + } + } + resolved.extend(structural); resolved } @@ -447,6 +504,7 @@ mod tests { delimiter: Some('/'), role: None, subscribed: false, + selectable: true, status: None, }, ResolvedFolder { @@ -457,6 +515,7 @@ mod tests { delimiter: Some('/'), role: Some("inbox"), subscribed: true, + selectable: true, status: None, }, ResolvedFolder { @@ -467,6 +526,7 @@ mod tests { delimiter: Some('/'), role: None, subscribed: false, + selectable: true, status: None, }, ]; @@ -508,4 +568,131 @@ mod tests { let folders = collect_from_list(&resps, false).unwrap(); assert_eq!(folders.len(), 1); } + + fn find<'a>(res: &'a [ResolvedFolder], name: &str) -> &'a ResolvedFolder { + res.iter().find(|f| f.name == name).unwrap() + } + + #[test] + fn noselect_parent_is_kept_as_structural() { + let folders = collect_from_list( + &[ + lst("INBOX", "/", &["\\HasChildren"]), + lst("INBOX/Projects", "/", &["\\Noselect", "\\HasChildren"]), + lst("INBOX/Projects/Invoices", "/", &["\\HasNoChildren"]), + ], + false, + ) + .unwrap(); + let mut res = apply_filters(folders, &filters_default()); + sort_by_depth(&mut res); + let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect(); + assert_eq!( + names, + ["INBOX", "INBOX/Projects", "INBOX/Projects/Invoices"] + ); + let projects = find(&res, "INBOX/Projects"); + assert!(!projects.selectable); + assert_eq!(projects.role, None); + assert_eq!(projects.leaf, "Projects"); + assert_eq!(projects.parent_path.as_deref(), Some("INBOX")); + assert!(find(&res, "INBOX/Projects/Invoices").selectable); + } + + #[test] + fn noselect_without_kept_children_is_dropped() { + let folders = collect_from_list( + &[lst("INBOX", "/", &[]), lst("Empty", "/", &["\\Noselect"])], + false, + ) + .unwrap(); + let res = apply_filters(folders, &filters_default()); + assert_eq!(res.len(), 1); + } + + #[test] + fn consecutive_structural_ancestors_and_synthesised_ones() { + let folders = collect_from_list( + &[lst("A", "/", &["\\NonExistent"]), lst("A/B/C/D", "/", &[])], + false, + ) + .unwrap(); + let mut res = apply_filters(folders, &filters_default()); + sort_by_depth(&mut res); + let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, ["A", "A/B", "A/B/C", "A/B/C/D"]); + assert!( + res.iter() + .take(3) + .all(|f| !f.selectable && f.role.is_none()) + ); + assert_eq!(find(&res, "A/B/C").wire_name, "A/B/C"); + assert_eq!(find(&res, "A/B").leaf, "B"); + } + + #[test] + fn trailing_space_and_same_leaf_under_different_parents() { + let folders = collect_from_list( + &[ + lst("Work ", ".", &["\\Noselect"]), + lst("Work", ".", &[]), + lst("Work .Archive", ".", &[]), + lst("Work.Archive", ".", &[]), + ], + false, + ) + .unwrap(); + let res = apply_filters(folders, &filters_default()); + assert_eq!(res.len(), 4); + let spaced = find(&res, "Work "); + assert!(!spaced.selectable); + assert_eq!(spaced.leaf, "Work "); + assert_eq!( + find(&res, "Work .Archive").parent_path.as_deref(), + Some("Work ") + ); + assert_eq!( + find(&res, "Work.Archive").parent_path.as_deref(), + Some("Work") + ); + assert!(find(&res, "Work").selectable); + } + + #[test] + fn synthesised_ancestor_derives_wire_name_from_child() { + let folders = collect_from_list(&[lst("&ZeVnLIqe-/Sub", "/", &[])], false).unwrap(); + let res = apply_filters(folders, &filters_default()); + let parent = find(&res, "日本語"); + assert_eq!(parent.wire_name, "&ZeVnLIqe-"); + assert!(!parent.selectable); + } + + #[test] + fn filters_keep_needed_ancestors_as_structural() { + let list = [ + lst("INBOX", "/", &[]), + lst("Projects", "/", &[]), + lst("Projects/Alpha", "/", &["\\Noselect"]), + lst("Projects/Alpha/Docs", "/", &["\\Subscribed"]), + lst("Other", "/", &[]), + ]; + let check = |res: Vec| { + assert_eq!(res.len(), 3, "{res:?}"); + assert!(!find(&res, "Projects").selectable); + assert!(!find(&res, "Projects/Alpha").selectable); + assert!(find(&res, "Projects/Alpha/Docs").selectable); + }; + let mut f = filters_default(); + f.include = vec![Regex::new("Docs$").unwrap()]; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + let mut f = filters_default(); + f.exclude = vec![Regex::new("^(INBOX|Other|Projects)$").unwrap()]; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + let mut f = filters_default(); + f.explicit = vec!["Projects/Alpha/Docs".to_owned()]; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + let mut f = filters_default(); + f.subscribed_only = true; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + } } diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs index d9e96a1..b3a7ae9 100644 --- a/src/sync/import_jmap.rs +++ b/src/sync/import_jmap.rs @@ -1,5 +1,6 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC * * SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -26,14 +27,19 @@ use crate::jmap::blobxfer; use crate::jmap::connect::{self, Connected}; use crate::jmap::error::JmapError; use crate::jmap::http::{Auth, HttpClient}; -use crate::jmap::request::{get_all, get_changes, get_objects, get_state, query_all_ids}; +use crate::jmap::request::{ + URN_VACATION_RESPONSE, get_all, get_changes, get_objects, get_state, query_all_ids, +}; use crate::jmap::session::{Limits, Session}; use crate::jmap::wire::JmapId; use crate::jmap::wire::common::parse_utc_date; use crate::jmap::wire::email::Email; -use crate::jmap::wire::file_node::{FileNode, NodeType}; +use crate::jmap::wire::file_node::{FileNode, FileProps, NodeKind}; use crate::jmap::wire::sieve_script::SieveScript; +use crate::jmap::wire::vacation_response::{self, VacationResponse}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::keys::index_to_json; use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts}; use crate::types::ObjectType; @@ -65,13 +71,18 @@ fn get_props(ty: ObjectType) -> Option<&'static [&'static str]> { ObjectType::Mailbox => Some(&["name", "parentId", "role", "sortOrder", "isSubscribed"]), ObjectType::Email => Some(&["blobId", "receivedAt", "mailboxIds", "keywords"]), ObjectType::SieveScript => Some(&["name", "isActive", "blobId"]), + // Both shapes' names (wire::file_node::Shape): each server ignores + // the other's. ObjectType::FileNode => Some(&[ - "parentId", + "@type", "nodeType", + "parentId", "blobId", "target", "name", + "mediaType", "type", + "executable", "created", "modified", "isSubscribed", @@ -107,17 +118,33 @@ impl LocalResolver for DbResolver<'_> { } } -struct PrefetchedBlobs<'a> { - conn: &'a Connection, - bytes: &'a HashMap>, +struct EmailMeta { + match_key: String, + date_header: Option, } -impl BlobIntern for PrefetchedBlobs<'_> { +struct StoredBlob { + local_id: i64, + email: Option, +} + +#[derive(Default)] +struct StoredBlobs(HashMap); + +impl StoredBlobs { + fn get(&self, jmap_blob_id: &str, kind: &str) -> Result<&StoredBlob, JmapError> { + self.0 + .get(jmap_blob_id) + .ok_or_else(|| JmapError::malformed(format!("{kind} blob missing"))) + } +} + +impl BlobIntern for &StoredBlobs { fn intern(&mut self, jmap_blob_id: &str) -> Result { - let data = self.bytes.get(jmap_blob_id).ok_or_else(|| { - JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched")) - })?; - Ok(db::blobs::intern_blob(self.conn, data)?) + self.0 + .get(jmap_blob_id) + .map(|b| b.local_id) + .ok_or_else(|| JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched"))) } } @@ -242,6 +269,34 @@ fn run_into( summary.per_type.push((ty.jmap_name(), counts)); } + if let Some(source_id) = source_id + && !ctx.dry_run() + && config + .objects + .as_ref() + .is_none_or(|s| s.contains(&ObjectType::SieveScript)) + && net.session.may_support(&net.account, URN_VACATION_RESPONSE) + { + if logger.enabled(LEVEL_DEFAULT) { + eprintln!("import: {} ...", vacation_response::TYPE_NAME); + } + let mut counts = TypeCounts::default(); + match import_vacation_response(&ctx.conn, &net, source_id) { + Ok(fetched) => counts.fetched = fetched, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!( + "type {} aborted: {e}; continuing (run will exit 5)", + vacation_response::TYPE_NAME + )); + counts.failed += 1; + } + } + summary + .per_type + .push((vacation_response::TYPE_NAME, counts)); + } + if ctx.dry_run() { print_dry_run(&dry_rows); *summary = Summary::default(); @@ -259,6 +314,28 @@ fn run_into( Ok(()) } +fn import_vacation_response(conn: &Connection, net: &Net, source_id: i64) -> Result { + let got = get_all::( + &net.client, + &net.api, + &net.account, + vacation_response::TYPE_NAME, + ) + .map_err(Error::from)?; + match got.list.into_iter().find(|v| !v.is_unset()) { + Some(vacation) => { + db::vacation_response::store(conn, source_id, &vacation) + .map_err(|e| Error::Partial(e.to_string()))?; + Ok(1) + } + None => { + db::vacation_response::clear(conn, source_id) + .map_err(|e| Error::Partial(e.to_string()))?; + Ok(0) + } + } +} + fn work_list(config: &ImportConfig, connected: &Connected) -> Vec { let selected = config.objects.as_ref(); IMPORT_ORDER @@ -462,9 +539,9 @@ fn insert_objects( for batch in ordered.chunks(200) { let blob_refs = blob_references(ty, batch.iter().map(|&i| &objects[i])); let blobs = if blob_refs.is_empty() { - HashMap::new() + StoredBlobs::default() } else { - download_blobs(net, blob_refs, threads, logger) + download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)? }; let tx = ctx .conn @@ -532,7 +609,8 @@ fn blob_references<'a>(ty: ObjectType, objects: impl Iterator) && seen.insert(b.to_owned()) { let mt = o - .get("type") + .get("mediaType") + .or_else(|| o.get("type")) .and_then(Value::as_str) .unwrap_or("application/octet-stream") .to_owned(); @@ -562,11 +640,13 @@ fn blob_references<'a>(ty: ObjectType, objects: impl Iterator) } fn download_blobs( + conn: &Connection, net: &Net, + ty: ObjectType, refs: Vec, threads: usize, logger: &Logger, -) -> HashMap> { +) -> Result { let workers = effective_workers(threads, &net.limits, true); let net_arc = Arc::new(net.clone()); let pool: Pool = Pool::new(workers, { @@ -587,11 +667,23 @@ fn download_blobs( for r in refs { pool.submit(r); } - let mut out = HashMap::new(); - for (blob_id, res) in pool.finish() { + let mut out = HashMap::with_capacity(total); + for _ in 0..total { + let Ok((blob_id, res)) = pool.results().recv() else { + break; + }; match res { Ok(bytes) => { - out.insert(blob_id, bytes); + let local_id = db::blobs::intern_blob(conn, &bytes) + .map_err(|e| Error::Partial(e.to_string()))?; + let email = (ty == ObjectType::Email).then(|| { + let (index, date_header) = email_meta_from_blob(&bytes); + EmailMeta { + match_key: index_to_json(&index), + date_header, + } + }); + out.insert(blob_id, StoredBlob { local_id, email }); } Err(e) => { logger.warn(&format!("blob {blob_id} download failed: {e}")); @@ -601,7 +693,7 @@ fn download_blobs( if logger.enabled(LEVEL_PROGRESS) { eprintln!("import: downloaded {}/{} blobs", out.len(), total); } - out + Ok(StoredBlobs(out)) } fn order_objects( @@ -649,7 +741,7 @@ fn insert_one( ty: ObjectType, source_id: i64, obj: &Value, - blobs: &HashMap>, + blobs: &StoredBlobs, logger: &Logger, ) -> Result { let resolver = DbResolver { conn, source_id }; @@ -676,14 +768,13 @@ fn insert_one( } ObjectType::Email => { let mut w: Email = serde_json::from_value(obj.clone())?; - let data = blobs - .get(&w.blob_id.0) + let stored = blobs.get(&w.blob_id.0, "email")?; + let meta = stored + .email + .as_ref() .ok_or_else(|| JmapError::malformed("email blob missing"))?; - let blob_local = db::blobs::intern_blob(conn, data)?; - let (index, date_header) = crate::sync::emailmeta::email_meta_from_blob(data); - let mm = crate::sync::keys::index_to_json(&index); if w.received_at.is_none() { - w.received_at = date_header.as_deref().and_then(parse_utc_date); + w.received_at = meta.date_header.as_deref().and_then(parse_utc_date); let id = obj.get("id").and_then(Value::as_str).unwrap_or("?"); let raw = obj .get("receivedAt") @@ -697,37 +788,31 @@ fn insert_one( } )); } - mapping::insert_email(conn, &w, blob_local, &mm, &resolver) + mapping::insert_email(conn, &w, stored.local_id, &meta.match_key, &resolver) } ObjectType::SieveScript => { let w: SieveScript = serde_json::from_value(obj.clone())?; - let data = blobs - .get(&w.blob_id.0) - .ok_or_else(|| JmapError::malformed("sieve blob missing"))?; - let blob_local = db::blobs::intern_blob(conn, data)?; + let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id; mapping::insert_sieve_script(conn, &w, blob_local) } ObjectType::FileNode => { let w: FileNode = serde_json::from_value(obj.clone())?; - let blob_local = match (w.effective_node_type(), &w.blob_id) { - (NodeType::File, Some(b)) => { - let data = blobs - .get(&b.0) - .ok_or_else(|| JmapError::malformed("file blob missing"))?; - Some(db::blobs::intern_blob(conn, data)?) - } + let blob_local = match &w.kind { + NodeKind::File(FileProps { + blob_id: Some(b), .. + }) => Some(blobs.get(&b.0, "file")?.local_id), _ => None, }; mapping::insert_file_node(conn, &w, blob_local, &resolver) } ObjectType::ContactCard => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::insert_contact_card(conn, &w, &resolver, &mut bi) } ObjectType::CalendarEvent => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::insert_calendar_event(conn, &w, &resolver, &mut bi) } } @@ -876,9 +961,9 @@ fn update_objects( for batch in objects.chunks(200) { let blob_refs = blob_references(ty, batch.iter()); let blobs = if blob_refs.is_empty() { - HashMap::new() + StoredBlobs::default() } else { - download_blobs(net, blob_refs, threads, logger) + download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)? }; let tx = ctx .conn @@ -915,7 +1000,7 @@ fn update_one( source_id: i64, local_id: i64, obj: &Value, - blobs: &HashMap>, + blobs: &StoredBlobs, ) -> Result { let resolver = DbResolver { conn, source_id }; match ty { @@ -942,33 +1027,27 @@ fn update_one( ObjectType::Email => mapping::update_email(conn, local_id, obj, &resolver), ObjectType::SieveScript => { let w: SieveScript = serde_json::from_value(obj.clone())?; - let data = blobs - .get(&w.blob_id.0) - .ok_or_else(|| JmapError::malformed("sieve blob missing"))?; - let blob_local = db::blobs::intern_blob(conn, data)?; + let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id; mapping::update_sieve_script(conn, local_id, &w, blob_local) } ObjectType::FileNode => { let w: FileNode = serde_json::from_value(obj.clone())?; - let blob_local = match (w.effective_node_type(), &w.blob_id) { - (NodeType::File, Some(b)) => { - let data = blobs - .get(&b.0) - .ok_or_else(|| JmapError::malformed("file blob missing"))?; - Some(db::blobs::intern_blob(conn, data)?) - } + let blob_local = match &w.kind { + NodeKind::File(FileProps { + blob_id: Some(b), .. + }) => Some(blobs.get(&b.0, "file")?.local_id), _ => None, }; mapping::update_file_node(conn, local_id, &w, blob_local, &resolver) } ObjectType::ContactCard => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::update_contact_card(conn, local_id, &w, &resolver, &mut bi) } ObjectType::CalendarEvent => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::update_calendar_event(conn, local_id, &w, &resolver, &mut bi) } } diff --git a/src/sync/import_jmap/mapping.rs b/src/sync/import_jmap/mapping.rs index 78ca2ee..ce9327f 100644 --- a/src/sync/import_jmap/mapping.rs +++ b/src/sync/import_jmap/mapping.rs @@ -17,7 +17,7 @@ use crate::jmap::wire::calendar::Calendar; use crate::jmap::wire::calendar_event::CalendarEvent; use crate::jmap::wire::contact_card::ContactCard; use crate::jmap::wire::email::Email; -use crate::jmap::wire::file_node::FileNode; +use crate::jmap::wire::file_node::{DirectoryProps, FileNode, FileProps, NodeKind, SymLinkProps}; use crate::jmap::wire::identity::Identity; use crate::jmap::wire::mailbox::Mailbox; use crate::jmap::wire::participant_identity::ParticipantIdentity; @@ -318,36 +318,62 @@ pub fn insert_file_node( resolver: &impl LocalResolver, ) -> Result { let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); - let node_type = serde_json::to_value(wire.effective_node_type())? - .as_str() - .unwrap_or("file") - .to_owned(); - let target = match &wire.target { - Some(t) => Some(serde_json::to_string(t)?), - None => None, - }; + let cols = FileNodeColumns::new(&wire.kind)?; conn.execute( "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + created, modified, is_subscribed, role, executable) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", params![ parent, - node_type, + wire.kind.db_name(), blob_local_id, - target, + cols.target, wire.name, - wire.media_type, + cols.media_type, format_utc(&wire.created)?, opt_format_utc(&wire.modified)?, - wire.is_subscribed as i64, - wire.role + cols.is_subscribed as i64, + cols.role, + cols.executable as i64 ], )?; Ok(conn.last_insert_rowid()) } +struct FileNodeColumns<'a> { + target: Option, + media_type: Option<&'a str>, + is_subscribed: bool, + role: Option<&'a str>, + executable: bool, +} + +impl<'a> FileNodeColumns<'a> { + fn new(kind: &'a NodeKind) -> Result { + let mut cols = FileNodeColumns { + target: None, + media_type: None, + is_subscribed: true, + role: None, + executable: false, + }; + match kind { + NodeKind::File(f) => { + cols.media_type = f.media_type.as_deref(); + cols.executable = f.executable; + } + NodeKind::Directory(d) => { + cols.role = d.role.as_deref(); + cols.is_subscribed = d.is_subscribed; + } + NodeKind::SymLink(l) => cols.target = Some(serde_json::to_string(&l.target)?), + } + Ok(cols) + } +} + pub const FILE_NODE_SELECT: &str = "SELECT id, parent_id, node_type, blob_id, target, name, \ - media_type, created, modified, is_subscribed, role FROM file_nodes"; + media_type, created, modified, is_subscribed, role, executable FROM file_nodes"; pub fn insert_contact_card( conn: &Connection, @@ -587,31 +613,27 @@ pub fn update_file_node( resolver: &impl LocalResolver, ) -> Result { let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); - let node_type = serde_json::to_value(wire.effective_node_type())? - .as_str() - .unwrap_or("file") - .to_owned(); - let target = match &wire.target { - Some(t) => Some(serde_json::to_string(t)?), - None => None, - }; + let cols = FileNodeColumns::new(&wire.kind)?; let n = conn.execute( "UPDATE file_nodes SET parent_id = ?1, node_type = ?2, blob_id = ?3, target = ?4, - name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, role = ?10 - WHERE id = ?11 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3 + name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, + role = ?10, executable = ?11 + WHERE id = ?12 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3 OR target IS NOT ?4 OR name IS NOT ?5 OR media_type IS NOT ?6 OR created IS NOT ?7 - OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10)", + OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10 + OR executable IS NOT ?11)", params![ parent, - node_type, + wire.kind.db_name(), blob_local_id, - target, + cols.target, wire.name, - wire.media_type, + cols.media_type, format_utc(&wire.created)?, opt_format_utc(&wire.modified)?, - wire.is_subscribed as i64, - wire.role, + cols.is_subscribed as i64, + cols.role, + cols.executable as i64, local_id ], )?; @@ -845,26 +867,36 @@ pub fn row_to_file_node( ), None => None, }; - let node_type = Some(serde_json::from_value(Value::from( - row.get::<_, String>(2)?, - ))?); - let target: Option> = from_opt_json(row.get::<_, Option>(4)?)?; - let blob_local_id: Option = row.get(3)?; + let node_type: String = row.get(2)?; + let kind = match node_type.as_str() { + "file" => NodeKind::File(FileProps { + blob_id: None, + media_type: row.get(6)?, + executable: row.get::<_, i64>(11)? != 0, + }), + "directory" => NodeKind::Directory(DirectoryProps { + role: row.get(10)?, + is_subscribed: row.get::<_, i64>(9)? != 0, + }), + "symlink" => NodeKind::SymLink(SymLinkProps { + target: from_opt_json(row.get::<_, Option>(4)?)?.unwrap_or_default(), + }), + other => { + return Err(JmapError::malformed(format!( + "unknown file_node type {other}" + ))); + } + }; Ok(FileNodeRow { wire: FileNode { id: None, parent_id, - node_type, - blob_id: None, - target, name: row.get(5)?, - media_type: row.get(6)?, created: parse_utc(&row.get::<_, String>(7)?)?, modified: opt_parse_utc(row.get::<_, Option>(8)?)?, - is_subscribed: row.get::<_, i64>(9)? != 0, - role: row.get(10)?, + kind, }, - blob_local_id, + blob_local_id: row.get(3)?, }) } @@ -1301,7 +1333,7 @@ mod tests { }; let node: FileNode = serde_json::from_value(serde_json::json!({ "id": "F1", - "nodeType": "symlink", + "@type": "SymLink", "target": ["..", "etc", "hosts"], "name": "link", "created": "2022-02-02T02:02:02Z" @@ -1410,7 +1442,7 @@ mod tests { to_target: HashMap::new(), }; let dir: FileNode = serde_json::from_value(serde_json::json!({ - "id": "D1", "nodeType": "directory", "name": "Docs", + "id": "D1", "@type": "Directory", "name": "Docs", "created": "2022-02-02T02:02:02Z" })) .unwrap(); @@ -1418,8 +1450,8 @@ mod tests { res.to_local .insert((ObjectType::FileNode, "D1".to_owned()), dirl); let file: FileNode = serde_json::from_value(serde_json::json!({ - "id": "F1", "nodeType": "file", "parentId": "D1", "name": "a.bin", - "type": "application/octet-stream", "created": "2022-02-02T02:02:02Z" + "id": "F1", "@type": "File", "parentId": "D1", "name": "a.bin", + "mediaType": "application/octet-stream", "created": "2022-02-02T02:02:02Z" })) .unwrap(); let blob = crate::db::blobs::intern_blob(&c, b"bytes").unwrap(); @@ -1439,6 +1471,72 @@ mod tests { assert_eq!(got.blob_local_id, Some(blob)); } + // Each @type serializes only the properties the draft defines for it. + #[test] + fn file_node_row_to_wire_emits_only_type_specific_properties() { + let c = mem(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let nodes = [ + serde_json::json!({"id": "D1", "@type": "Directory", "name": "Docs", + "role": "documents", "isSubscribed": false, + "created": "2022-02-02T02:02:02Z"}), + serde_json::json!({"id": "F1", "@type": "File", "name": "run.sh", + "blobId": "B1", "mediaType": "text/x-sh", "executable": true, "size": 3, + "created": "2022-02-02T02:02:02Z"}), + serde_json::json!({"id": "L1", "@type": "SymLink", "name": "latest", + "target": ["", "Docs"], "created": "2022-02-02T02:02:02Z"}), + ]; + let blob = crate::db::blobs::intern_blob(&c, b"#!\n").unwrap(); + let mut out = Vec::new(); + for (i, n) in nodes.into_iter().enumerate() { + let w: FileNode = serde_json::from_value(n).unwrap(); + let blob_local = matches!(w.kind, NodeKind::File(_)).then_some(blob); + let l = insert_file_node(&c, &w, blob_local, &res).unwrap(); + res.to_target + .insert((ObjectType::FileNode, l), format!("T{i}")); + let row = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![l], + |r| Ok(row_to_file_node(r, &res)), + ) + .unwrap() + .unwrap(); + out.push(serde_json::to_value(&row.wire).unwrap()); + } + let keys = |v: &Value| { + let mut k: Vec = v.as_object().unwrap().keys().cloned().collect(); + k.sort_unstable(); + k + }; + assert_eq!( + keys(&out[0]), + ["@type", "created", "isSubscribed", "name", "role"] + ); + assert_eq!(out[0]["@type"], "Directory"); + assert_eq!(out[0]["isSubscribed"], false); + assert_eq!( + keys(&out[1]), + ["@type", "created", "executable", "mediaType", "name"] + ); + assert_eq!(out[1]["executable"], true); + assert_eq!(out[1]["mediaType"], "text/x-sh"); + assert_eq!(keys(&out[2]), ["@type", "created", "name", "target"]); + assert_eq!(out[2]["@type"], "SymLink"); + assert_eq!(out[2]["target"], serde_json::json!(["", "Docs"])); + } + + #[test] + fn file_node_rejects_unknown_type() { + let r: Result = serde_json::from_value(serde_json::json!({ + "id": "X1", "@type": "Socket", "name": "s", "created": "2022-02-02T02:02:02Z" + })); + assert!(r.is_err()); + } + fn one_row(c: &Connection, table: &str) -> i64 { c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) .unwrap() @@ -1653,30 +1751,31 @@ mod tests { to_target: HashMap::new(), }; let file: FileNode = serde_json::from_value(serde_json::json!({ - "id":"F1","nodeType":"file","name":"a.bin", - "type":"application/octet-stream","created":"2022-02-02T02:02:02Z" + "id":"F1","@type":"File","name":"a.bin", + "mediaType":"application/octet-stream","created":"2022-02-02T02:02:02Z" })) .unwrap(); let blob1 = crate::db::blobs::intern_blob(&c, b"v1").unwrap(); let local = insert_file_node(&c, &file, Some(blob1), &res).unwrap(); let blob2 = crate::db::blobs::intern_blob(&c, b"v2").unwrap(); let changed: FileNode = serde_json::from_value(serde_json::json!({ - "id":"F1","nodeType":"file","name":"renamed.bin", - "type":"text/plain","created":"2022-02-02T02:02:02Z" + "id":"F1","@type":"File","name":"renamed.bin", + "mediaType":"text/plain","executable":true,"created":"2022-02-02T02:02:02Z" })) .unwrap(); update_file_node(&c, local, &changed, Some(blob2), &res).unwrap(); assert_eq!(one_row(&c, "file_nodes"), 1); - let (name, mt, b): (String, Option, Option) = c + let (name, mt, b, x): (String, Option, Option, i64) = c .query_row( - "SELECT name, media_type, blob_id FROM file_nodes WHERE id=?1", + "SELECT name, media_type, blob_id, executable FROM file_nodes WHERE id=?1", params![local], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), ) .unwrap(); assert_eq!(name, "renamed.bin"); assert_eq!(mt.as_deref(), Some("text/plain")); assert_eq!(b, Some(blob2)); + assert_eq!(x, 1); } #[test] diff --git a/src/sync/import_jmap/pool.rs b/src/sync/import_jmap/pool.rs index ca86bbc..bb58a12 100644 --- a/src/sync/import_jmap/pool.rs +++ b/src/sync/import_jmap/pool.rs @@ -6,7 +6,7 @@ use std::thread::JoinHandle; -use crossbeam_channel::{Receiver, Sender, unbounded}; +use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; use crate::jmap::session::Limits; @@ -41,7 +41,7 @@ where { let threads = threads.max(1); let (job_tx, job_rx) = unbounded::(); - let (result_tx, result_rx) = unbounded::(); + let (result_tx, result_rx) = bounded::(threads); let worker = std::sync::Arc::new(worker); let mut workers = Vec::with_capacity(threads); for _ in 0..threads { @@ -75,10 +75,11 @@ where pub fn finish(mut self) -> Vec { drop(self.job_tx.take()); + let out = self.result_rx.iter().collect(); for handle in self.workers.drain(..) { let _ = handle.join(); } - self.result_rx.try_iter().collect() + out } } @@ -89,6 +90,7 @@ where { fn drop(&mut self) { drop(self.job_tx.take()); + for _ in self.result_rx.iter() {} for handle in self.workers.drain(..) { let _ = handle.join(); } @@ -131,4 +133,38 @@ mod tests { let expected: Vec = (0..100).map(|n| n * n).collect(); assert_eq!(got, expected); } + + #[test] + fn pool_holds_at_most_one_unread_result_per_worker() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + // Unread results back-pressure the workers instead of piling up. + let done = std::sync::Arc::new(AtomicUsize::new(0)); + let pool: Pool = Pool::new(2, { + let done = done.clone(); + move |n| { + done.fetch_add(1, Ordering::SeqCst); + n + } + }); + for n in 0..50 { + pool.submit(n); + } + std::thread::sleep(Duration::from_millis(200)); + assert!(done.load(Ordering::SeqCst) <= 4); + let mut got = pool.finish(); + got.sort_unstable(); + assert_eq!(got, (0..50).collect::>()); + } + + #[test] + fn dropping_pool_with_unread_results_does_not_hang() { + let pool: Pool = Pool::new(2, |n| n); + for n in 0..50 { + pool.submit(n); + } + let _ = pool.results().recv(); + drop(pool); + } } diff --git a/tests/integration/dovecot.rs b/tests/integration/dovecot.rs index f862e0a..321e39e 100644 --- a/tests/integration/dovecot.rs +++ b/tests/integration/dovecot.rs @@ -359,6 +359,23 @@ impl Dovecot { Ok(()) } + pub fn create_leaves_without_parents( + &self, + account: &Account, + leaves: &[&str], + message: &[u8], + ) -> ContainerResult> { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + for name in leaves { + client.create(name)?; + client.append_with_flags(name, &[], message)?; + } + let listed = client.list_raw()?; + client.logout()?; + Ok(listed) + } + pub fn delete_first_inbox_message(&self, account: &Account) -> ContainerResult<()> { let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; client.login(&account.username, &account.password)?; diff --git a/tests/integration/imap_client.rs b/tests/integration/imap_client.rs index a2a770a..3e2b23b 100644 --- a/tests/integration/imap_client.rs +++ b/tests/integration/imap_client.rs @@ -189,6 +189,12 @@ impl ImapSeed { Ok(names) } + pub fn list_raw(&mut self) -> ContainerResult> { + let tag = self.next_tag(); + self.send(&format!(r#"{tag} LIST "" "*""#))?; + self.read_tagged(&tag) + } + pub fn select(&mut self, mailbox: &str) -> ContainerResult { let tag = self.next_tag(); self.send(&format!("{tag} SELECT {}", quote(mailbox)))?; diff --git a/tests/integration/inbuxa.rs b/tests/integration/inbuxa.rs index 742d9d8..598d51a 100644 --- a/tests/integration/inbuxa.rs +++ b/tests/integration/inbuxa.rs @@ -243,6 +243,7 @@ fn jmap(base_url: &str, auth: &str, calls: Value) -> ContainerResult { "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail", "urn:ietf:params:jmap:sieve", + "urn:ietf:params:jmap:filenode", "urn:inbuxa:jmap:registry", ], "methodCalls": calls, diff --git a/tests/integration_baikal.rs b/tests/integration_baikal.rs index a96bb99..5aec926 100644 --- a/tests/integration_baikal.rs +++ b/tests/integration_baikal.rs @@ -19,7 +19,7 @@ use integration::validate::{ use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::import_dav; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use rusqlite::Connection; use serde_json::Value; @@ -39,7 +39,7 @@ fn dav_config(kind: DavKindArg, account: &Account, dav_root: &str) -> DavImportC allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } @@ -58,7 +58,7 @@ fn dav_config_via_discovery( allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } @@ -412,7 +412,7 @@ fn baikal_carddav_preserves_apple_item_labels() { allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, }; let summary = import_dav::run(common(&archive), cfg).expect("carddav import"); assert!( diff --git a/tests/integration_export_dav.rs b/tests/integration_export_dav.rs index bcc9090..dc1a839 100644 --- a/tests/integration_export_dav.rs +++ b/tests/integration_export_dav.rs @@ -44,7 +44,7 @@ fn import(kind: DavKindArg, url: String, account: &Account, archive: &Path) -> S allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject, }, ) .expect("import"); diff --git a/tests/integration_inbuxa.rs b/tests/integration_inbuxa.rs index f8b2ee4..66f1635 100644 --- a/tests/integration_inbuxa.rs +++ b/tests/integration_inbuxa.rs @@ -367,3 +367,90 @@ fn the_guide_sets_up_the_accounts_previews_and_moves_them() { "{report}" ); } + +#[test] +#[ignore = "requires Docker"] +fn files_go_into_inbuxa_in_its_own_shape_and_come_back() { + let server = Inbuxa::start().expect("start inbuxa"); + let dir = TempDir::new().unwrap(); + let out = dir.path().join("out.sqlite"); + let back = dir.path().join("back.sqlite"); + { + let conn = db::init::open(&out).unwrap(); + let blob = db::blobs::intern_blob(&conn, b"#!/bin/sh\necho hi\n").unwrap(); + conn.execute( + "INSERT INTO file_nodes (id,parent_id,node_type,blob_id,name,media_type,created) + VALUES (1,NULL,'directory',NULL,'docs',NULL,'2022-02-02T02:02:02Z'), + (2,1,'file',?1,'run.sh','text/x-sh','2022-02-02T02:02:02Z')", + [blob], + ) + .unwrap(); + } + let run = |args: &[&str]| { + let out = Command::new(BIN) + .args(args) + .env("INBUXA_MIGRATE_API_KEY", &server.token) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + assert!(out.status.success(), "{args:?}: {stderr}"); + }; + let target = [ + "--url", + server.base_url.as_str(), + "--auth-bearer", + "--account-name", + ADMIN_EMAIL, + "--objects", + "filenode", + ]; + run(&[&["export"][..], &target, &[out.to_str().unwrap()]].concat()); + + let account = server.account_of(ADMIN_EMAIL, ADMIN_PASSWORD).unwrap(); + let r = server + .call( + ADMIN_EMAIL, + ADMIN_PASSWORD, + json!([["FileNode/query", {"accountId": account}, "q"], + ["FileNode/get", {"accountId": account, + "#ids": {"resultOf": "q", "name": "FileNode/query", "path": "/ids"}, + "properties": ["name", "nodeType", "type", "parentId"]}, "g"]]), + ) + .unwrap(); + let list = r[1][1]["list"].as_array().unwrap_or_else(|| panic!("{r}")); + let file = list + .iter() + .find(|n| n["name"] == "run.sh") + .unwrap_or_else(|| panic!("no file: {r}")); + assert_eq!(file["nodeType"], "file", "{file}"); + assert_eq!(file["type"], "text/x-sh", "{file}"); + assert!( + list.iter() + .any(|n| n["name"] == "docs" && n["nodeType"] == "directory"), + "{r}" + ); + + run(&[&["import", "jmap"][..], &target, &[back.to_str().unwrap()]].concat()); + let conn = db::init::open(&back).unwrap(); + let mut rows: Vec<(String, String, Option)> = conn + .prepare( + "SELECT name, node_type, media_type FROM file_nodes WHERE name IN ('docs','run.sh')", + ) + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap() + .collect::>() + .unwrap(); + rows.sort(); + assert_eq!( + rows, + [ + ("docs".to_owned(), "directory".to_owned(), None), + ( + "run.sh".to_owned(), + "file".to_owned(), + Some("text/x-sh".to_owned()) + ) + ] + ); +} diff --git a/tests/integration_radicale.rs b/tests/integration_radicale.rs index 06165e7..d79007e 100644 --- a/tests/integration_radicale.rs +++ b/tests/integration_radicale.rs @@ -19,7 +19,7 @@ use integration::{Account, Endpoint}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::import_dav; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use rusqlite::Connection; use serde_json::Value; @@ -34,7 +34,7 @@ fn dav_config(kind: DavKindArg, account: &Account, http: &Endpoint) -> DavImport allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/integration_webdav.rs b/tests/integration_webdav.rs index 68885bb..12e7ecb 100644 --- a/tests/integration_webdav.rs +++ b/tests/integration_webdav.rs @@ -18,7 +18,7 @@ use integration::{Account, Endpoint}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::import_dav; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig { DavImportConfig { @@ -31,7 +31,7 @@ fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig { allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/mock_dav.rs b/tests/mock_dav.rs index dc681d2..d3fdb75 100644 --- a/tests/mock_dav.rs +++ b/tests/mock_dav.rs @@ -1340,7 +1340,9 @@ fn webdav_display_or_basename_test_via_parser_keeps_basename_semantics() { fn dry_run_writes_nothing_but_emits_per_collection_counts() { use inbuxa_migrate::logging::Logger; use inbuxa_migrate::sync::CommonConfig; - use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; + use inbuxa_migrate::sync::import_dav::{ + DavAuth, DavImportConfig, DavKindArg, SourceChange, run, + }; use std::path::PathBuf; let mut server = mockito::Server::new(); @@ -1409,7 +1411,7 @@ fn dry_run_writes_nothing_but_emits_per_collection_counts() { allow_cleartext: true, dav_connections: 1, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, }; let summary = run(common, config).expect("dry-run ok"); @@ -1453,7 +1455,9 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { use encodify::base64::STANDARD; use inbuxa_migrate::logging::Logger; use inbuxa_migrate::sync::CommonConfig; - use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; + use inbuxa_migrate::sync::import_dav::{ + DavAuth, DavImportConfig, DavKindArg, SourceChange, run, + }; use std::path::PathBuf; let mut server = mockito::Server::new(); @@ -1568,7 +1572,7 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { allow_cleartext: true, dav_connections: 1, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, }; run( @@ -1698,7 +1702,7 @@ fn caldav_import_config(url: &str) -> inbuxa_migrate::sync::import_dav::DavImpor allow_cleartext: true, dav_connections: 1, multiget_batch: 8, - allow_source_change: false, + source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject, } } @@ -1830,6 +1834,122 @@ END:VCALENDAR let _ = std::fs::remove_file(&archive); } +// Kerio Connect sends calendar-data with bare CR line breaks, raw or as (issue #48). +#[test] +fn kerio_bare_cr_calendar_data_imports() { + let mut server = mockito::Server::new(); + let url = server.url(); + let home = "/full-calendars/domain.ext/user/"; + let coll = "/full-calendars/domain.ext/user/ceb087f2-0604-4c31-a70e-9f68e90600c6/"; + + let home_set = format!( + r#" + + + {coll} + + + + Calendar + + HTTP/1.1 200 OK + + +"# + ); + let _discovery = multistatus_response(&mut server, "PROPFIND", home, &home_set); + + let item = |n: u32| format!("{coll}ceb087f2-0604-4c31-a70e-9f68e90600c6-0000000{n}.eml"); + let listing = format!( + r#" + + + {} + "k1" + HTTP/1.1 200 OK + + + {} + "k2" + HTTP/1.1 200 OK + +"#, + item(1), + item(2) + ); + let _enumeration = multistatus_response(&mut server, "PROPFIND", coll, &listing); + + let ical = |uid: &str, summary: &str, eol: &str| { + [ + "BEGIN:VCALENDAR", + "PRODID:-//Kerio Technologies//Kerio Connect//EN", + "VERSION:2.0", + "BEGIN:VEVENT", + &format!("UID:{uid}"), + "DTSTAMP:20260101T000000Z", + "DTSTART:20260101T090000Z", + "DTEND:20260101T100000Z", + &format!("SUMMARY:{summary}"), + "X-LABEL:0", + "END:VEVENT", + "END:VCALENDAR", + "", + ] + .join(eol) + }; + let data = format!( + r#" + + + {} + "k1" + + HTTP/1.1 200 OK + + + {} + "k2" + {} + HTTP/1.1 200 OK + +"#, + item(1), + ical("k1@kerio", "Raw CR", "\r"), + item(2), + ical("k2@kerio", "Entity CR", " "), + ); + let _multiget = multistatus_response(&mut server, "REPORT", coll, &data); + + let archive = dav_archive(); + let summary = inbuxa_migrate::sync::import_dav::run( + dav_common(&archive), + caldav_import_config(&format!("{url}{home}")), + ) + .expect("import"); + + let events = summary + .per_type + .iter() + .find(|(t, _)| *t == "calendarevent") + .map(|(_, c)| c.clone()) + .expect("calendarevent counts"); + assert_eq!(events.failed, 0, "{events:?}"); + assert_eq!(events.fetched, 2, "{events:?}"); + + let conn = rusqlite::Connection::open(&archive).expect("open archive"); + let mut titles: Vec = conn + .prepare("SELECT json_extract(data, '$.title') FROM calendar_events") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + titles.sort_unstable(); + assert_eq!(titles, ["Entity CR", "Raw CR"]); + drop(conn); + let _ = std::fs::remove_file(&archive); +} + #[test] fn credentials_rejected_during_discovery_still_aborts_the_run() { let mut server = mockito::Server::new(); @@ -1904,3 +2024,190 @@ fn dav_import_reports_both_phases_when_every_collection_fails() { ); let _ = std::fs::remove_file(&archive); } + +fn webdav_share(server: &mut mockito::Server, prefix: &str, body: &str) -> Vec { + let url = server.url(); + let root = format!( + r#" + + + {url}{prefix} + + share + HTTP/1.1 200 OK + + + + {url}{prefix}docs/ + + docs + HTTP/1.1 200 OK + + +"# + ); + let docs = format!( + r#" + + + {url}{prefix}docs/ + + docs + HTTP/1.1 200 OK + + + + {url}{prefix}docs/a.txt + + text/plain"a1" + HTTP/1.1 200 OK + + +"# + ); + vec![ + multistatus_response(server, "PROPFIND", prefix, &root), + multistatus_response(server, "PROPFIND", &format!("{prefix}docs/"), &docs), + server + .mock("GET", format!("{prefix}docs/a.txt").as_str()) + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(body) + .create(), + ] +} + +fn webdav_import_config( + url: &str, + source_change: inbuxa_migrate::sync::import_dav::SourceChange, +) -> inbuxa_migrate::sync::import_dav::DavImportConfig { + inbuxa_migrate::sync::import_dav::DavImportConfig { + kind: inbuxa_migrate::sync::import_dav::DavKindArg::Webdav, + source_change, + ..caldav_import_config(url) + } +} + +#[test] +fn allow_source_change_keeps_the_moved_share_old_tree() { + use inbuxa_migrate::sync::import_dav::{SourceChange, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let _old = webdav_share(&mut server, "/old/u/", "old\n"); + let _new = webdav_share(&mut server, "/new/u/", "new\n"); + let archive = dav_archive(); + + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject), + ) + .expect("old share import"); + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Allow), + ) + .expect("new share import"); + + let (nodes, sources) = ( + row_count(&archive, "file_nodes"), + row_count(&archive, "sources"), + ); + let _ = std::fs::remove_file(&archive); + assert_eq!(nodes, 4, "both trees stay side by side"); + assert_eq!(sources, 2); +} + +#[test] +fn replace_source_drops_the_moved_share_old_tree() { + use inbuxa_migrate::sync::import_dav::{SourceChange, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let _old = webdav_share(&mut server, "/old/u/", "old\n"); + let _new = webdav_share(&mut server, "/new/u/", "new\n"); + let archive = dav_archive(); + + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject), + ) + .expect("old share import"); + let summary = run( + dav_common(&archive), + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Replace), + ) + .expect("new share import"); + let rerun = run( + dav_common(&archive), + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Reject), + ); + + let conn = rusqlite::Connection::open(&archive).unwrap(); + let account: String = conn + .query_row("SELECT account_id FROM sources", [], |r| r.get(0)) + .unwrap(); + let orphans: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes + WHERE id NOT IN (SELECT local_id FROM sync_id_dav)", + [], + |r| r.get(0), + ) + .unwrap(); + drop(conn); + let (nodes, sources, blobs) = ( + row_count(&archive, "file_nodes"), + row_count(&archive, "sources"), + row_count(&archive, "blobs"), + ); + let _ = std::fs::remove_file(&archive); + + assert_eq!(nodes, 2, "only the new tree remains"); + assert_eq!(orphans, 0); + assert_eq!(sources, 1); + assert!(account.ends_with("/new/u/"), "kept source is {account}"); + assert_eq!(blobs, 1, "the old file's blob is collected"); + let filenode = summary + .per_type + .iter() + .find(|(name, _)| *name == "filenode") + .expect("filenode counts"); + assert_eq!(filenode.1.deleted, 2); + assert!( + rerun.is_ok(), + "once replaced, the archive records a single source: {rerun:?}" + ); +} + +#[test] +fn dry_run_replace_source_keeps_the_old_tree() { + use inbuxa_migrate::sync::import_dav::{SourceChange, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let _old = webdav_share(&mut server, "/old/u/", "old\n"); + let _new = webdav_share(&mut server, "/new/u/", "new\n"); + let archive = dav_archive(); + + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject), + ) + .expect("old share import"); + let mut common = dav_common(&archive); + common.dry_run = true; + run( + common, + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Replace), + ) + .expect("dry-run"); + + let (nodes, sources) = ( + row_count(&archive, "file_nodes"), + row_count(&archive, "sources"), + ); + let _ = std::fs::remove_file(&archive); + assert_eq!(nodes, 2); + assert_eq!(sources, 1); +} diff --git a/tests/mock_dav_occurrences.rs b/tests/mock_dav_occurrences.rs index e1c255a..3669ccf 100644 --- a/tests/mock_dav_occurrences.rs +++ b/tests/mock_dav_occurrences.rs @@ -70,7 +70,7 @@ fn import(url: &str, archive: &Path) -> Summary { allow_cleartext: true, dav_connections: 1, multiget_batch: 25, - allow_source_change: false, + source_change: import_dav::SourceChange::Reject, }, ) .expect("import"); diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs index 42b0a24..0a1eaec 100644 --- a/tests/mock_imap.rs +++ b/tests/mock_imap.rs @@ -2394,3 +2394,248 @@ fn coordinator_signs_in_to_gmail_as_a_service_account_acting_for_the_user() { // Both connections signed in on the one token. _token.assert(); } + +const NOSELECT_LIST: &[&str] = &[ + "* LIST (\\HasChildren) \"/\" \"INBOX\"", + "* LIST (\\Noselect \\HasChildren) \"/\" \"INBOX/Projects\"", + "* LIST (\\HasNoChildren) \"/\" \"INBOX/Projects/Invoices\"", + "* LIST (\\Noselect \\HasChildren) \"/\" \"A\"", + "* LIST (\\HasNoChildren) \"/\" \"A/B/Reports\"", + "* LIST (\\Noselect \\HasChildren) \"/\" \"Clients \"", + "* LIST (\\HasNoChildren) \"/\" \"Clients /Reports\"", + "* LIST (\\HasChildren) \"/\" \"Clients\"", + "* LIST (\\HasNoChildren) \"/\" \"Clients/Reports\"", +]; + +const NOSELECT_SELECTABLE: &[&str] = &[ + "INBOX", + "INBOX/Projects/Invoices", + "A/B/Reports", + "Clients /Reports", + "Clients", + "Clients/Reports", +]; + +fn noselect_control_script(selected: std::sync::Arc>>) -> Script { + Box::new(move |conn: &mut MockConn| -> std::io::Result<()> { + auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?; + while let Ok((tag, cmd)) = conn.read_command() { + if cmd.starts_with("LIST") { + for line in NOSELECT_LIST { + conn.write_line(line)?; + } + conn.write_line(&format!("{tag} OK LIST done"))?; + } else if let Some(name) = cmd + .strip_prefix("SELECT ") + .or_else(|| cmd.strip_prefix("EXAMINE ")) + { + selected + .lock() + .expect("selected") + .push(name.trim_matches('"').to_owned()); + write_select(conn, &tag, 7, 1, 0)?; + } else if cmd.starts_with("UID SEARCH") { + conn.write_line("* SEARCH")?; + conn.write_line(&format!("{tag} OK SEARCH done"))?; + } else if cmd.starts_with("LOGOUT") { + conn.write_line("* BYE bye")?; + conn.write_line(&format!("{tag} OK"))?; + break; + } else { + assert!( + !cmd.starts_with("STATUS") && !cmd.contains("FETCH"), + "unexpected {cmd}" + ); + conn.write_line(&format!("{tag} OK"))?; + } + } + Ok(()) + }) +} + +fn mailbox_tree(conn: &Connection) -> Vec<(i64, String, Option)> { + let mut stmt = conn + .prepare( + "SELECT m.id, s.folder, p.name FROM mailboxes m + JOIN sync_id_imap s ON s.local_id = m.id AND s.type_name = 'mailbox' + LEFT JOIN mailboxes p ON p.id = m.parent_id + ORDER BY s.folder", + ) + .unwrap(); + stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap() + .collect::>() + .unwrap() +} + +fn assert_noselect_tree(conn: &Connection) { + let tree = mailbox_tree(conn); + let pairs: Vec<(&str, Option<&str>)> = tree + .iter() + .map(|(_, f, p)| (f.as_str(), p.as_deref())) + .collect(); + assert_eq!( + pairs, + [ + ("A", None), + ("A/B", Some("A")), + ("A/B/Reports", Some("B")), + ("Clients", None), + ("Clients ", None), + ("Clients /Reports", Some("Clients ")), + ("Clients/Reports", Some("Clients")), + ("INBOX", None), + ("INBOX/Projects", Some("INBOX")), + ("INBOX/Projects/Invoices", Some("Projects")), + ] + ); + let roles: i64 = conn + .query_row( + "SELECT COUNT(*) FROM mailboxes WHERE role IS NOT NULL AND name IN ('Projects', 'A', 'B', 'Clients ')", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(roles, 0); +} + +fn assert_only_selectable_selected(selected: &Mutex>) { + let mut got = selected.lock().expect("selected").clone(); + got.sort_unstable(); + let mut want: Vec = NOSELECT_SELECTABLE + .iter() + .map(|s| (*s).to_owned()) + .collect(); + want.sort_unstable(); + assert_eq!(got, want); +} + +#[test] +fn coordinator_keeps_noselect_ancestors_and_never_selects_them() { + let first = std::sync::Arc::new(Mutex::new(Vec::new())); + let second = std::sync::Arc::new(Mutex::new(Vec::new())); + let server = MockImap::start_scripts(vec![ + noselect_control_script(first.clone()), + noselect_control_script(second.clone()), + ]); + let archive = tempfile("noselect"); + let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("import"); + let mailbox = summary + .per_type + .iter() + .find(|(k, _)| *k == "mailbox") + .unwrap(); + assert_eq!(mailbox.1.created, 10); + assert_eq!(mailbox.1.failed, 0); + assert_only_selectable_selected(&first); + let before = { + let conn = Connection::open(&archive).unwrap(); + assert_noselect_tree(&conn); + mailbox_tree(&conn) + }; + + let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("reimport"); + let mailbox = summary + .per_type + .iter() + .find(|(k, _)| *k == "mailbox") + .unwrap(); + assert_eq!((mailbox.1.created, mailbox.1.deleted), (0, 0)); + assert_only_selectable_selected(&second); + let conn = Connection::open(&archive).unwrap(); + assert_eq!(mailbox_tree(&conn), before); +} + +#[test] +fn coordinator_heals_archive_missing_noselect_parents() { + let first = std::sync::Arc::new(Mutex::new(Vec::new())); + let second = std::sync::Arc::new(Mutex::new(Vec::new())); + let server = MockImap::start_scripts(vec![ + noselect_control_script(first), + noselect_control_script(second), + ]); + let archive = tempfile("noselect_heal"); + run_import(&server, "alice", archive.clone(), |_| {}).expect("import"); + let child_id = { + let conn = Connection::open(&archive).unwrap(); + let tree = mailbox_tree(&conn); + let child = tree + .iter() + .find(|(_, f, _)| f == "INBOX/Projects/Invoices") + .unwrap() + .0; + // simulate an archive written before structural parents were kept + conn.execute( + "UPDATE mailboxes SET parent_id = NULL WHERE id = ?1", + [child], + ) + .unwrap(); + let parent: i64 = conn + .query_row( + "SELECT local_id FROM sync_id_imap WHERE type_name = 'mailbox' AND folder = 'INBOX/Projects'", + [], + |r| r.get(0), + ) + .unwrap(); + conn.execute( + "DELETE FROM sync_id_imap WHERE type_name = 'mailbox' AND folder = 'INBOX/Projects'", + [], + ) + .unwrap(); + conn.execute("DELETE FROM mailboxes WHERE id = ?1", [parent]) + .unwrap(); + child + }; + run_import(&server, "alice", archive.clone(), |_| {}).expect("reimport"); + let conn = Connection::open(&archive).unwrap(); + assert_noselect_tree(&conn); + let healed = mailbox_tree(&conn) + .into_iter() + .find(|(_, f, _)| f == "INBOX/Projects/Invoices") + .unwrap() + .0; + assert_eq!(healed, child_id); +} + +#[test] +fn dry_run_never_selects_noselect_ancestors() { + let selected = std::sync::Arc::new(Mutex::new(Vec::new())); + let server = MockImap::start_scripts(vec![noselect_control_script(selected.clone())]); + let archive = tempfile("noselect_dry"); + let common = CommonConfig { + archive: archive.clone(), + threads: 1, + dry_run: true, + max_retries: 1, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + }; + let config = ImapImportConfig { + url: server.url(), + auth: ImapAuth::Basic { + user: "alice".to_owned(), + password: "p@ss".to_owned(), + }, + allow_cleartext: true, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 256, + fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, + imap_connections: 1, + allow_source_change: false, + }; + let summary = run(common, config).expect("dry run"); + let mailbox = summary + .per_type + .iter() + .find(|(k, _)| *k == "mailbox") + .unwrap(); + assert_eq!(mailbox.1.created, 10); + assert_only_selectable_selected(&selected); +} diff --git a/tests/mock_import_carddav_books.rs b/tests/mock_import_carddav_books.rs index 6482b53..95e7f9f 100644 --- a/tests/mock_import_carddav_books.rs +++ b/tests/mock_import_carddav_books.rs @@ -44,7 +44,7 @@ fn import(url: &str, archive: &Path) -> Summary { allow_cleartext: true, dav_connections: 1, multiget_batch: 25, - allow_source_change: false, + source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject, }, ) .expect("import"); diff --git a/tests/mock_jmap.rs b/tests/mock_jmap.rs index c03e9ce..dc65e06 100644 --- a/tests/mock_jmap.rs +++ b/tests/mock_jmap.rs @@ -333,8 +333,9 @@ fn principal_substring_prefilter_rejects_near_matches() { ["Principal/get", { "list": [ { "id": "p1", "name": "alice2", "accounts": {} }, { "id": "p2", "name": "alice", "accounts": { - "w": { "urn:ietf:params:jmap:principals:owner": - { "accountIdForPrincipal": "w" } } } } + "w": { "name": "alice", "accountCapabilities": { + "urn:ietf:params:jmap:principals:owner": + { "accountIdForPrincipal": "admin", "principalId": "p2" } } } } } ] }, "g"] ] }) .to_string(), diff --git a/tests/mock_sync.rs b/tests/mock_sync.rs index f528466..cc71f47 100644 --- a/tests/mock_sync.rs +++ b/tests/mock_sync.rs @@ -5187,3 +5187,1005 @@ fn export_rerun_leaves_an_event_alone_when_nothing_is_newer() { ); assert_eq!(counts.skipped, 1); } + +// Draft -18 FileNodes import by @type; an unrecognised @type fails only that node. +#[test] +fn import_file_node_by_type_and_skips_unknown_type() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _fnterm = anchor_terminator(&mut server, api, "FileNode"); + + let _fq = server + .mock("POST", api) + .match_body(Matcher::Regex("FileNode/query".into())) + .with_body( + json!({"methodResponses":[["FileNode/query", + {"accountId":"w","ids":["F1","F2","F3","F4"]},"q"]]}) + .to_string(), + ) + .create(); + let _fg = server + .mock("POST", api) + .match_body(Matcher::Regex("FileNode/get".into())) + .with_body( + json!({"methodResponses":[["FileNode/get",{"accountId":"w","state":"sf1","list":[ + {"@type":"Directory","id":"F1","parentId":null,"name":"docs","role":null, + "isSubscribed":true,"created":"2026-01-01T00:00:00Z"}, + {"@type":"File","id":"F2","parentId":"F1","name":"a.sh","blobId":"BLBF", + "size":10,"mediaType":"text/x-sh","executable":true, + "created":"2026-01-02T00:00:00Z"}, + {"@type":"SymLink","id":"F3","parentId":null,"name":"latest", + "target":["docs","a.sh"],"created":"2026-01-03T00:00:00Z"}, + {"@type":"Socket","id":"F4","parentId":null,"name":"sock", + "created":"2026-01-04T00:00:00Z"} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .create(); + let _dl = server + .mock("GET", Matcher::Regex("/jmap/dl/w/BLBF/.*".into())) + .with_body("file bytes") + .create(); + + let outcome = sync::import_jmap::run_reporting( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::FileNode]), + ); + assert!(outcome.error.is_none(), "{:?}", outcome.error); + + let nodes = outcome + .summary + .per_type + .iter() + .find(|(t, _)| *t == "FileNode") + .map(|(_, c)| c.clone()) + .expect("file node counts"); + assert_eq!(nodes.fetched, 3); + assert_eq!(nodes.failed, 1); + + let conn = rusqlite::Connection::open(&archive).unwrap(); + let mut stmt = conn + .prepare( + "SELECT name, node_type, media_type, executable, target FROM file_nodes ORDER BY name", + ) + .unwrap(); + type Row = (String, String, Option, i64, Option); + let rows: Vec = stmt + .query_map([], |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)) + }) + .unwrap() + .map(Result::unwrap) + .collect(); + drop(stmt); + assert_eq!( + rows, + vec![ + ( + "a.sh".to_owned(), + "file".to_owned(), + Some("text/x-sh".to_owned()), + 1, + None + ), + ("docs".to_owned(), "directory".to_owned(), None, 0, None), + ( + "latest".to_owned(), + "symlink".to_owned(), + None, + 0, + Some(r#"["docs","a.sh"]"#.to_owned()) + ), + ] + ); + let _ = std::fs::remove_file(&archive); +} + +#[test] +fn import_writes_each_email_blob_before_later_downloads_complete() { + // Each download waits for the previous blob to reach the archive WAL; collecting a page first would time out. + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + let wal = PathBuf::from(format!("{}-wal", archive.display())); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _mbterm = anchor_terminator(&mut server, api, "Mailbox"); + let _emterm = anchor_terminator(&mut server, api, "Email"); + let _mbq = server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/query".into())) + .with_body( + json!({"methodResponses":[["Mailbox/query", + {"accountId":"w","ids":["MX"]},"q"]]}) + .to_string(), + ) + .create(); + let _mbg = server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/get".into())) + .with_body( + json!({"methodResponses":[["Mailbox/get",{"accountId":"w","state":"sm1","list":[ + {"id":"MX","name":"Inbox","parentId":null,"role":"inbox","sortOrder":0,"isSubscribed":true} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .create(); + let _eq = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/query".into())) + .with_body( + json!({"methodResponses":[["Email/query", + {"accountId":"w","ids":["E1","E2","E3"]},"q"]]}) + .to_string(), + ) + .create(); + let list: Vec = (1..=3) + .map(|n| { + json!({"id":format!("E{n}"),"blobId":format!("BLB{n}"), + "receivedAt":"2020-01-01T00:00:00Z","mailboxIds":{"MX":true},"keywords":{}}) + }) + .collect(); + let _eg = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/get".into())) + .with_body( + json!({"methodResponses":[["Email/get",{"accountId":"w","state":"se1", + "list":list,"notFound":[]},"g"]]}) + .to_string(), + ) + .create(); + + let wal_seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let wal_len = { + let wal = wal.clone(); + move || std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0) + }; + let mut downloads = Vec::new(); + for n in 1..=3u8 { + let seen = wal_seen.clone(); + let wal_len = wal_len.clone(); + downloads.push( + server + .mock("GET", Matcher::Regex(format!("/jmap/dl/w/BLB{n}/.*"))) + .with_body_from_request(move |_| { + let floor = seen.lock().unwrap().last().copied(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + let mut len = wal_len(); + while floor.is_some_and(|f| len <= f) && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + len = wal_len(); + } + seen.lock().unwrap().push(len); + let mut body = format!("From: a@x\r\nMessage-ID: <{n}@h>\r\n\r\n").into_bytes(); + body.resize(body.len() + 64 * 1024, b'a' + n); + body + }) + .create(), + ); + } + + let summary = sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]), + ) + .expect("import succeeds"); + assert!(!summary.any_failed()); + + let seen = wal_seen.lock().unwrap().clone(); + assert_eq!(seen.len(), 3); + assert!( + seen.windows(2).all(|w| w[1] > w[0]), + "each blob reaches the archive before the next download finishes: {seen:?}" + ); + let conn = rusqlite::Connection::open(&archive).unwrap(); + assert_eq!( + conn.query_row::("SELECT count(*) FROM emails", [], |r| r.get(0)) + .unwrap(), + 3 + ); + let _ = std::fs::remove_file(&archive); +} + +// DAV alpha colors are sent without alpha and unusable colors are dropped. +#[test] +fn export_calendar_normalizes_colors() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO calendars (id,name,color,is_default) VALUES + (1,'Sogo','#AAAAAAFF',0), (2,'Short','#abcd',0), (3,'Odd','rgba(1,2,3,0.5)',0)", + [], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _g = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/get".into())) + .with_body( + json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[], + "notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let create = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/set".into())) + .with_body_from_request(|req| { + let body: Value = serde_json::from_slice(req.body().unwrap()).unwrap(); + let create = &body["methodCalls"][0][1]["create"]; + assert_eq!(create["c1"]["color"], json!("#AAAAAA")); + assert_eq!(create["c2"]["color"], json!("#abc")); + assert!(create["c3"].get("color").is_none(), "{create}"); + json!({"methodResponses":[["Calendar/set",{"accountId":"w", + "created":{"c1":{"id":"T1"},"c2":{"id":"T2"},"c3":{"id":"T3"}}},"s"]]}) + .to_string() + .into_bytes() + }) + .expect(1) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::Calendar]), + ) + .expect("export"); + create.assert(); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "Calendar") + .map(|(_, c)| c.clone()) + .expect("calendar counts"); + assert_eq!(counts.created, 3); + assert_eq!(counts.failed, 0); + let _ = std::fs::remove_file(&archive); +} + +// Archived participant identities are never sent to the target, even with prune, and nothing fails. +#[test] +fn export_skips_participant_identities() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO participant_identities (id,name,calendar_address,is_default) + VALUES (1,'Bob','mailto:bob@example.org',1)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO calendars (id,name,is_default) VALUES (1,'Family',1)", + [], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let participant = server + .mock("POST", api) + .match_body(Matcher::Regex("ParticipantIdentity/".into())) + .with_status(500) + .expect(0) + .create(); + let calendars = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/get".into())) + .with_body( + json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[ + {"id":"F","name":"Family","isDefault":true} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + + for objects in [ + vec![ObjectType::ParticipantIdentity, ObjectType::Calendar], + vec![ObjectType::ParticipantIdentity], + ] { + let mut config = export_cfg_objects(&base, objects); + config.prune = true; + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + assert!( + summary + .per_type + .iter() + .all(|(t, _)| *t != "ParticipantIdentity") + ); + } + participant.assert(); + calendars.assert(); + let _ = std::fs::remove_file(&archive); +} + +// Pruning stale scripts leaves the archive's active script active on the target. +#[test] +fn export_sieve_prune_keeps_archive_active_script_active() { + prune_sieve_with_archive_active(true, Some("M1")); +} + +// With no active script in the archive, the stale active target script is deactivated and pruned. +#[test] +fn export_sieve_prune_destroys_stale_active_script() { + prune_sieve_with_archive_active(false, None); +} + +fn prune_sieve_with_archive_active(archive_active: bool, expected_active: Option<&str>) { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + let blob = db::blobs::intern_blob(&conn, b"keep;\n").unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'main',?1,?2)", + rusqlite::params![archive_active, blob], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let active = std::sync::Arc::new(std::sync::Mutex::new(Some("O1".to_owned()))); + let target_active = active.clone(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, move |name, args| { + let mut active = target_active.lock().unwrap(); + match name { + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"M1","name":"main","isActive":active.as_deref() == Some("M1"),"blobId":"BM"}, + {"id":"O1","name":"old","isActive":active.as_deref() == Some("O1"),"blobId":"BO"} + ]}), + "SieveScript/set" => { + if args["onSuccessDeactivateScript"] == json!(true) { + *active = None; + } + if let Some(id) = args["onSuccessActivateScript"].as_str() { + *active = Some(id.to_owned()); + } + let mut response = json!({"accountId":"w"}); + if let Some(ids) = args["destroy"].as_array() { + let (blocked, destroyed): (Vec<&Value>, Vec<&Value>) = + ids.iter().partition(|id| id.as_str() == active.as_deref()); + response["destroyed"] = json!(destroyed); + response["notDestroyed"] = blocked + .iter() + .map(|id| { + ( + id.as_str().unwrap().to_owned(), + json!({"type":"scriptIsActive"}), + ) + }) + .collect::>() + .into(); + } + response + } + other => panic!("unexpected {other}"), + } + }); + + let mut config = export_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.prune = true; + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let destroyed: Vec = log + .lock() + .unwrap() + .iter() + .filter_map(|(_, args)| args.get("destroy").cloned()) + .collect(); + assert_eq!(destroyed, vec![json!(["O1"])]); + assert_eq!(active.lock().unwrap().as_deref(), expected_active); + let _ = std::fs::remove_file(&archive); +} + +fn session_body_vacation(base: &str) -> String { + let mut session: Value = serde_json::from_str(&session_body_full(base)).unwrap(); + session["accounts"]["w"]["accountCapabilities"]["urn:ietf:params:jmap:vacationresponse"] = + json!({}); + session.to_string() +} + +type CallLog = std::sync::Arc>>; + +fn jmap_dispatch( + server: &mut mockito::Server, + log: &CallLog, + respond: impl Fn(&str, &Value) -> Value + Send + Sync + 'static, +) -> mockito::Mock { + let log = log.clone(); + server + .mock("POST", "/jmap/api") + .with_body_from_request(move |req| { + let body: Value = serde_json::from_slice(req.body().unwrap()).unwrap(); + let mut responses = Vec::new(); + for call in body["methodCalls"].as_array().unwrap() { + let name = call[0].as_str().unwrap(); + log.lock().unwrap().push((name.to_owned(), call[1].clone())); + responses.push(json!([name, respond(name, &call[1]), call[2]])); + } + json!({ "methodResponses": responses }) + .to_string() + .into_bytes() + }) + .expect_at_least(1) + .create() +} + +fn vacation_row(archive: &Path) -> Option<(bool, Option, Option)> { + let conn = db::init::open(archive).unwrap(); + conn.query_row( + "SELECT is_enabled, subject, from_date FROM vacation_response", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .ok() +} + +// A 0.16 source keeps the vacation script and stores the VacationResponse; catch-up refreshes it. +#[test] +fn import_vacation_response_from_stalwart_016_source() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + let vacation = std::sync::Arc::new(std::sync::Mutex::new(json!({ + "id": "singleton", "isEnabled": true, "fromDate": "2026-10-01T00:00:00Z", + "toDate": null, "subject": "Away", "textBody": "Back soon", "htmlBody": null + }))); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_vacation(&base)) + .expect_at_least(1) + .create(); + let _dl1 = server + .mock("GET", Matcher::Regex("/jmap/dl/w/B1/.*".into())) + .with_body("keep;\n") + .create(); + let _dl2 = server + .mock("GET", Matcher::Regex("/jmap/dl/w/B2/.*".into())) + .with_body("require [\"vacation\"];\nvacation \"Back soon\";\n") + .create(); + let log = CallLog::default(); + let current = vacation.clone(); + let _api = jmap_dispatch(&mut server, &log, move |name, args| match name { + "SieveScript/query" if args.get("anchor").is_some() => json!({"accountId":"w","ids":[]}), + "SieveScript/query" => json!({"accountId":"w","ids":["S1","S2"]}), + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"S1","name":"filters","isActive":false,"blobId":"B1"}, + {"id":"S2","name":"vacation","isActive":true,"blobId":"B2"} + ]}), + "VacationResponse/get" => json!({"accountId":"w","state":"x","notFound":[], + "list":[current.lock().unwrap().clone()]}), + other => panic!("unexpected {other}"), + }); + + let summary = sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("import"); + assert!(!summary.any_failed()); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "VacationResponse") + .map(|(_, c)| c.clone()) + .expect("vacation counts"); + assert_eq!(counts.fetched, 1); + assert_eq!( + vacation_row(&archive), + Some(( + true, + Some("Away".to_owned()), + Some("2026-10-01T00:00:00Z".to_owned()) + )) + ); + let scripts: i64 = db::init::open(&archive) + .unwrap() + .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0)) + .unwrap(); + assert_eq!(scripts, 2, "the vacation script is still archived"); + + *vacation.lock().unwrap() = json!({"id": "singleton", "isEnabled": false, "subject": "Later"}); + sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("catch-up import"); + assert_eq!( + vacation_row(&archive), + Some((false, Some("Later".to_owned()), None)) + ); + + *vacation.lock().unwrap() = json!({"id": "singleton", "isEnabled": false}); + sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("second catch-up import"); + assert_eq!( + vacation_row(&archive), + None, + "an unset vacation clears the row" + ); + let _ = std::fs::remove_file(&archive); +} + +fn seed_vacation_archive(archive: &Path) { + let conn = db::init::open(archive).unwrap(); + let filters = db::blobs::intern_blob(&conn, b"keep;\n").unwrap(); + let vacation = db::blobs::intern_blob(&conn, b"vacation \"Back soon\";\n").unwrap(); + conn.execute( + "INSERT INTO sources (id,kind,session_url,account_id,username) + VALUES (1,'jmap','http://src','a','bob')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES + (1,'filters',0,?1), (2,'vacation',1,?2)", + rusqlite::params![filters, vacation], + ) + .unwrap(); + conn.execute( + "INSERT INTO vacation_response (id,source_id,is_enabled,subject,text_body) + VALUES (1,1,1,'Away','Back soon')", + [], + ) + .unwrap(); +} + +// The vacation script becomes a VacationResponse update, sent last and never pruned. +#[test] +fn export_vacation_script_becomes_vacation_response() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_vacation_archive(&archive); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_vacation(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let second_run = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let target_state = second_run.clone(); + let _api = jmap_dispatch(&mut server, &log, move |name, args| match name { + "SieveScript/get" if target_state.load(std::sync::atomic::Ordering::Relaxed) => { + json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"V1","name":"vacation","isActive":true,"blobId":"BV"}, + {"id":"S1","name":"filters","isActive":false,"blobId":"UP1"} + ]}) + } + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"O1","name":"old","isActive":false,"blobId":"BO"} + ]}), + "SieveScript/set" if args.get("create").is_some() => { + json!({"accountId":"w","created":{"c1":{"id":"S1"}}}) + } + "SieveScript/set" if args.get("destroy").is_some() => { + json!({"accountId":"w","destroyed":args["destroy"].clone()}) + } + "SieveScript/set" => json!({"accountId":"w"}), + "VacationResponse/set" => json!({"accountId":"w","updated":{"singleton":null}}), + other => panic!("unexpected {other}"), + }); + let export = || { + let mut config = export_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.prune = true; + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let calls = std::mem::take(&mut *log.lock().unwrap()); + (summary, calls) + }; + let args_of = |calls: &[(String, Value)], key: &str| -> Vec { + calls + .iter() + .filter_map(|(_, args)| args.get(key).cloned()) + .collect() + }; + let expected_vacation = json!({"isEnabled": true, "fromDate": null, "toDate": null, + "subject": "Away", "textBody": "Back soon", "htmlBody": null}); + + let (summary, calls) = export(); + let creates = args_of(&calls, "create"); + assert_eq!(creates.len(), 1); + assert_eq!(creates[0]["c1"]["name"], json!("filters")); + assert!( + creates[0].get("c2").is_none(), + "vacation is not created as a script" + ); + assert_eq!(args_of(&calls, "destroy"), vec![json!(["O1"])]); + let (last_name, last_args) = calls.last().unwrap(); + assert_eq!(last_name, "VacationResponse/set"); + assert_eq!(last_args["update"]["singleton"], expected_vacation); + let counts = |ty: &str| { + summary + .per_type + .iter() + .find(|(t, _)| *t == ty) + .map(|(_, c)| c.clone()) + .unwrap() + }; + assert_eq!(counts("SieveScript").created, 1); + assert_eq!(counts("SieveScript").skipped, 1); + assert_eq!(counts("VacationResponse").updated, 1); + + second_run.store(true, std::sync::atomic::Ordering::Relaxed); + let (_, calls) = export(); + assert!(args_of(&calls, "create").is_empty()); + assert!( + args_of(&calls, "destroy").is_empty(), + "the target vacation script is not pruned" + ); + let (last_name, last_args) = calls.last().unwrap(); + assert_eq!(last_name, "VacationResponse/set"); + assert_eq!(last_args["update"]["singleton"], expected_vacation); + let _ = std::fs::remove_file(&archive); +} + +// Without VacationResponse on the target, the vacation script is exported as before. +#[test] +fn export_vacation_script_unchanged_without_target_capability() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_vacation_archive(&archive); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, |name, args| match name { + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[]}), + "SieveScript/set" if args.get("create").is_some() => { + let id = args["create"] + .as_object() + .unwrap() + .keys() + .next() + .unwrap() + .clone(); + json!({"accountId":"w","created":{ id.clone(): {"id": format!("T{id}")} }}) + } + "SieveScript/set" => json!({"accountId":"w"}), + other => panic!("unexpected {other}"), + }); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let calls = log.lock().unwrap().clone(); + let names: Vec<&Value> = calls + .iter() + .filter_map(|(_, args)| args.get("create")) + .flat_map(|c| c.as_object().unwrap().values().map(|v| &v["name"])) + .collect(); + assert_eq!(names, vec![&json!("filters"), &json!("vacation")]); + assert!( + calls + .iter() + .all(|(n, _)| !n.starts_with("VacationResponse")) + ); + assert!( + summary + .per_type + .iter() + .all(|(t, _)| *t != "VacationResponse") + ); + let _ = std::fs::remove_file(&archive); +} + +fn session_body_admin_vacation(base: &str) -> String { + let mut session: Value = serde_json::from_str(&session_body_full(base)).unwrap(); + let caps = &mut session["capabilities"]; + caps["urn:ietf:params:jmap:principals"] = json!({}); + caps["urn:ietf:params:jmap:sieve"] = json!({}); + caps["urn:ietf:params:jmap:vacationresponse"] = json!({}); + let mut admin = session["accounts"]["w"].take(); + admin["name"] = json!("admin"); + session["accounts"] = json!({ "a": admin }); + session.to_string() +} + +fn principal_bob(name: &str) -> Option { + match name { + "Principal/query" => Some(json!({"accountId":"a","ids":["p2"]})), + "Principal/get" => Some(json!({"accountId":"a","state":"x","notFound":[],"list":[ + {"id":"p2","name":"bob","accounts":{"w":{"name":"bob"}}} + ]})), + _ => None, + } +} + +// Admin credentials with --account-name: the target is not listed in the session, the session advertises VacationResponse. +#[test] +fn export_vacation_response_for_unlisted_account_of_admin_session() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_vacation_archive(&archive); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_admin_vacation(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/w/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, |name, args| { + if let Some(reply) = principal_bob(name) { + return reply; + } + match name { + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[]}), + "SieveScript/set" if args.get("create").is_some() => { + json!({"accountId":"w","created":{"c1":{"id":"S1"}}}) + } + "SieveScript/set" => json!({"accountId":"w"}), + "VacationResponse/set" => json!({"accountId":"w","updated":{"singleton":null}}), + other => panic!("unexpected {other}"), + } + }); + + let mut config = export_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.connect.account = AccountSelector::Name("bob".into()); + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let calls = log.lock().unwrap().clone(); + let names: Vec<&Value> = calls + .iter() + .filter_map(|(_, args)| args.get("create")) + .flat_map(|c| c.as_object().unwrap().values().map(|v| &v["name"])) + .collect(); + assert_eq!(names, vec![&json!("filters")], "vacation is not created"); + let (last_name, last_args) = calls.last().unwrap(); + assert_eq!(last_name, "VacationResponse/set"); + assert_eq!(last_args["accountId"], json!("w")); + assert_eq!(last_args["update"]["singleton"]["subject"], json!("Away")); + let _ = std::fs::remove_file(&archive); +} + +// Admin credentials with --account-name: the import stores the target account's VacationResponse. +#[test] +fn import_vacation_response_for_unlisted_account_of_admin_session() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_admin_vacation(&base)) + .expect_at_least(1) + .create(); + let _dl = server + .mock("GET", Matcher::Regex("/jmap/dl/w/B1/.*".into())) + .with_body("require [\"vacation\"];\nvacation \"Back soon\";\n") + .create(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, |name, args| { + if let Some(reply) = principal_bob(name) { + return reply; + } + match name { + "SieveScript/query" if args.get("anchor").is_some() => { + json!({"accountId":"w","ids":[]}) + } + "SieveScript/query" => json!({"accountId":"w","ids":["S1"]}), + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"S1","name":"vacation","isActive":true,"blobId":"B1"} + ]}), + "VacationResponse/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"singleton","isEnabled":true,"fromDate":null,"toDate":null, + "subject":"Away","textBody":"Back soon","htmlBody":null} + ]}), + other => panic!("unexpected {other}"), + } + }); + + let mut config = import_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.connect.account = AccountSelector::Name("bob".into()); + let summary = sync::import_jmap::run(common(&archive), config).expect("import"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let fetched = summary + .per_type + .iter() + .find(|(t, _)| *t == "VacationResponse") + .map(|(_, c)| c.fetched); + assert_eq!(fetched, Some(1)); + assert_eq!( + vacation_row(&archive), + Some((true, Some("Away".to_owned()), None)) + ); + assert!( + log.lock() + .unwrap() + .iter() + .any(|(n, args)| n == "VacationResponse/get" && args["accountId"] == json!("w")) + ); + let _ = std::fs::remove_file(&archive); +} + +// A target on the 0.16 line, as inbuxa is, answers the shape probe and gets +// `nodeType` and `type`; a draft-18 one refuses the probe's sort and gets +// `@type`, `mediaType` and `executable`. +#[test] +fn export_file_nodes_in_the_targets_own_shape() { + for legacy in [true, false] { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + let blob = db::blobs::intern_blob(&conn, b"#!/bin/sh\n").unwrap(); + conn.execute( + "INSERT INTO file_nodes (id,parent_id,node_type,blob_id,name,media_type,created,executable) + VALUES (1,NULL,'directory',NULL,'docs',NULL,'2022-02-02T02:02:02Z',0), + (2,1,'file',?1,'run.sh','text/x-sh','2022-02-02T02:02:02Z',1)", + [blob], + ) + .unwrap(); + } + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let seen = log.clone(); + let _api = server + .mock("POST", "/jmap/api") + .with_body_from_request(move |req| { + let body: Value = serde_json::from_slice(req.body().unwrap()).unwrap(); + let mut responses = Vec::new(); + for call in body["methodCalls"].as_array().unwrap() { + let (name, args) = (call[0].as_str().unwrap(), &call[1]); + seen.lock().unwrap().push((name.to_owned(), args.clone())); + let (name, reply) = match name { + "FileNode/query" if args.get("sort").is_some() && !legacy => { + ("error", json!({"type":"unsupportedSort"})) + } + "FileNode/query" => { + (name, json!({"accountId":"w","ids":[],"queryState":"q"})) + } + "FileNode/set" => { + let created: serde_json::Map = args["create"] + .as_object() + .unwrap() + .keys() + .enumerate() + .map(|(i, k)| (k.clone(), json!({"id": format!("N{i}{k}")}))) + .collect(); + (name, json!({"accountId":"w","created":created})) + } + other => panic!("unexpected {other}"), + }; + responses.push(json!([name, reply, call[2]])); + } + json!({ "methodResponses": responses }) + .to_string() + .into_bytes() + }) + .expect_at_least(1) + .create(); + + let config = export_cfg_objects(&base, vec![ObjectType::FileNode]); + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + + let created: Vec = log + .lock() + .unwrap() + .iter() + .filter(|(name, _)| name == "FileNode/set") + .flat_map(|(_, args)| { + args["create"] + .as_object() + .unwrap() + .values() + .cloned() + .collect::>() + }) + .collect(); + let file = created + .iter() + .find(|n| n["name"] == "run.sh") + .expect("file created"); + let dir = created + .iter() + .find(|n| n["name"] == "docs") + .expect("directory created"); + if legacy { + assert_eq!(dir["nodeType"], "directory", "{dir}"); + assert_eq!(file["nodeType"], "file", "{file}"); + assert_eq!(file["type"], "text/x-sh", "{file}"); + for n in [file, dir] { + for gone in ["@type", "mediaType", "executable"] { + assert!(n.get(gone).is_none(), "{gone} in {n}"); + } + } + } else { + assert_eq!(dir["@type"], "Directory", "{dir}"); + assert_eq!(file["@type"], "File", "{file}"); + assert_eq!(file["mediaType"], "text/x-sh", "{file}"); + assert_eq!(file["executable"], true, "{file}"); + for n in [file, dir] { + assert!( + n.get("nodeType").is_none() && n.get("type").is_none(), + "{n}" + ); + } + } + let _ = std::fs::remove_file(&archive); + } +} diff --git a/tests/seed_smoke.rs b/tests/seed_smoke.rs index 7b9ae45..fdb850c 100644 --- a/tests/seed_smoke.rs +++ b/tests/seed_smoke.rs @@ -268,14 +268,9 @@ fn provision_seed_verify_and_teardown() { let owner = exact[0] .get("accounts") .and_then(Value::as_object) - .and_then(|m| { - m.iter().find_map(|(_, v)| { - v.get("urn:ietf:params:jmap:principals:owner") - .and_then(|o| o.get("accountIdForPrincipal")) - .and_then(Value::as_str) - }) - }) - .expect("accountIdForPrincipal"); + .and_then(|m| m.keys().next()) + .map(String::as_str) + .expect("principal accounts key"); assert_eq!(owner, test1.account_id, "resolved account id matches test1"); seeder::teardown(stalwart.base_url()).expect("teardown"); diff --git a/tests/seeder/admin.rs b/tests/seeder/admin.rs index 5a3f954..7c591eb 100644 --- a/tests/seeder/admin.rs +++ b/tests/seeder/admin.rs @@ -121,6 +121,32 @@ impl Admin { Ok(()) } + pub fn enable_directory_queries(&self) -> SeedResult<()> { + let response = self.jmap.call( + STALWART, + "x:Sharing/set", + &self.account_id, + json!({ "update": { "singleton": { "allowDirectoryQueries": true } } }), + )?; + if response.pointer("/updated/singleton").is_none() { + return Err(SeedError::Shape(format!( + "x:Sharing/set not applied: {response}" + ))); + } + let response = self.jmap.call( + STALWART, + "x:Action/set", + &self.account_id, + json!({ "create": { "r": { "@type": "ReloadSettings" } } }), + )?; + if response.pointer("/created/r").is_none() { + return Err(SeedError::Shape(format!( + "ReloadSettings failed: {response}" + ))); + } + Ok(()) + } + pub fn teardown_domain(&self, name: &str) -> SeedResult<()> { let Some(domain_id) = self.domain_id(name)? else { return Ok(()); diff --git a/tests/seeder/mod.rs b/tests/seeder/mod.rs index 64cbe65..5490077 100644 --- a/tests/seeder/mod.rs +++ b/tests/seeder/mod.rs @@ -177,6 +177,7 @@ pub fn teardown(base_url: &str) -> SeedResult<()> { pub fn provision(base_url: &str) -> SeedResult { let admin = Admin::connect(base_url, ADMIN_USER, ADMIN_PASSWORD)?; + admin.enable_directory_queries()?; admin.teardown_domain(DOMAIN)?; admin.invalidate_caches()?; let domain_id = admin.ensure_domain(DOMAIN)?; diff --git a/tests/seeder/seed.rs b/tests/seeder/seed.rs index 62f1b28..edbe216 100644 --- a/tests/seeder/seed.rs +++ b/tests/seeder/seed.rs @@ -273,14 +273,14 @@ fn seed_filenodes(jmap: &Jmap, account_id: &str, tree: &[FileSpec]) -> SeedResul obj.insert("parentId".to_owned(), Value::String(format!("#{parent}"))); } if spec.directory { - obj.insert("nodeType".to_owned(), Value::String("directory".to_owned())); + obj.insert("@type".to_owned(), Value::String("Directory".to_owned())); } else { let bytes = random_bytes(128 + i * 17)?; let blob_id = jmap.upload(account_id, "application/octet-stream", &bytes)?; - obj.insert("nodeType".to_owned(), Value::String("file".to_owned())); + obj.insert("@type".to_owned(), Value::String("File".to_owned())); obj.insert("blobId".to_owned(), Value::String(blob_id)); obj.insert( - "type".to_owned(), + "mediaType".to_owned(), Value::String("application/octet-stream".to_owned()), ); } @@ -400,6 +400,9 @@ fn seed_contacts_calendars( for immutable in ["method", "prodId"] { obj.remove(immutable); } + if obj.contains_key("recurrenceRule") || obj.contains_key("recurrenceOverrides") { + obj.remove("recurrenceId"); + } let calendar = &calendars[i % calendars.len()]; obj.insert("calendarIds".to_owned(), json!({ calendar.as_str(): true })); jmap.set_create( diff --git a/tests/sync_caldav.rs b/tests/sync_caldav.rs index 5512540..fc07138 100644 --- a/tests/sync_caldav.rs +++ b/tests/sync_caldav.rs @@ -20,7 +20,7 @@ fn base_url() -> &'static str { use inbuxa_migrate::jmap::account::AccountSelector; use inbuxa_migrate::jmap::http::Auth; use inbuxa_migrate::logging::Logger; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { @@ -59,7 +59,7 @@ fn caldav_cfg(base_url: &str, domain: &str, localpart: &str) -> DavImportConfig allow_cleartext: false, dav_connections: 4, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/sync_carddav.rs b/tests/sync_carddav.rs index 7f531b6..fae7baa 100644 --- a/tests/sync_carddav.rs +++ b/tests/sync_carddav.rs @@ -20,7 +20,7 @@ use rusqlite::Connection; fn base_url() -> &'static str { shared_stalwart().base_url() } -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { @@ -59,7 +59,7 @@ fn carddav_cfg(base_url: &str, domain: &str, localpart: &str) -> DavImportConfig allow_cleartext: false, dav_connections: 4, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/sync_exchange_graph.rs b/tests/sync_exchange_graph.rs index 7b569b3..0d8391d 100644 --- a/tests/sync_exchange_graph.rs +++ b/tests/sync_exchange_graph.rs @@ -377,16 +377,28 @@ fn graph_import_survives_a_full_export_to_stalwart() { .find(|c| c["name"]["full"] == "Graph Contact") .expect("the imported card reached the target"); - let uri = card["media"] + let photo = card["media"] .as_object() .and_then(|m| m.values().next()) - .and_then(|m| m["uri"].as_str()) .expect("the contact photo survived as a Media resource"); - assert!( - uri.starts_with("data:image/png;base64,"), + let media_type = match photo["uri"].as_str() { + Some(uri) => uri + .strip_prefix("data:") + .and_then(|rest| rest.split_once(';')) + .map(|(mt, _)| mt), + None => { + assert!( + photo["blobId"].is_string(), + "the photo is either an inline data URI or an embedded blob: {photo}" + ); + photo["mediaType"].as_str() + } + }; + assert_eq!( + media_type, + Some("image/png"), "Graph reports image/jpeg for every contact photo, so the media type must come \ - from the bytes; got {}", - &uri[..uri.len().min(40)] + from the bytes; got {photo}" ); assert_eq!(card["keywords"]["work"], true, "categories became keywords"); assert_eq!( @@ -409,16 +421,24 @@ fn graph_import_survives_a_full_export_to_stalwart() { .iter() .find(|e| e["title"] == "Event with an enclosure") .expect("the event with an attachment reached the target"); - let href = enclosure["links"] + let link = enclosure["links"] .as_object() .and_then(|m| m.values().next()) - .and_then(|l| l["href"].as_str()) .expect("the attachment survived as a Link"); - assert!( - href.starts_with("data:text/plain;base64,"), - "got {}", - &href[..href.len().min(40)] - ); + match link["href"].as_str() { + Some(href) => assert!( + href.starts_with("data:text/plain;base64,"), + "got {}", + &href[..href.len().min(40)] + ), + None => { + assert!( + link["blobId"].is_string(), + "the attachment is either an inline data URI or an embedded blob: {link}" + ); + assert_eq!(link["contentType"], "text/plain", "got {link}"); + } + } let series = events .iter() @@ -449,7 +469,7 @@ fn graph_import_survives_a_full_export_to_stalwart() { assert!( files .iter() - .any(|f| f["name"] == "report.txt" && f["blobId"].is_string()), + .any(|f| f["name"] == "report.txt" && f["@type"] == "File" && f["blobId"].is_string()), "the drive file reached the target with a blob" ); assert!( diff --git a/tests/sync_imap.rs b/tests/sync_imap.rs index 1e136b1..658530f 100644 --- a/tests/sync_imap.rs +++ b/tests/sync_imap.rs @@ -8,11 +8,15 @@ mod integration; mod seeder; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; +use integration::Account; +use integration::dovecot::Dovecot; use integration::stalwart::shared as shared_stalwart; +use integration::validate::{emails_in_mailbox, mailbox_id_by_path, mailbox_path, open_archive}; use rusqlite::Connection; +use serde_json::json; fn base_url() -> &'static str { shared_stalwart().base_url() @@ -77,6 +81,29 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig { } } +fn dovecot_config(account: &Account, imap: &integration::Endpoint) -> ImapImportConfig { + ImapImportConfig { + url: format!("imap://{}:{}", imap.host, imap.port), + auth: ImapAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: false, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 64, + fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES, + imap_connections: 2, + allow_source_change: false, + } +} + fn count(conn: &Connection, table: &str) -> i64 { conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) .unwrap() @@ -395,3 +422,145 @@ fn imap_imported_archive_exports_via_jmap() { let _ = std::fs::remove_file(&archive); seeder::teardown(base_url()).expect("teardown"); } + +#[test] +#[ignore = "requires Docker"] +fn dovecot_noselect_parents_keep_the_tree_through_export() { + let d = Dovecot::start().expect("dovecot start"); + let account = d.accounts.first().expect("account").clone(); + let message = b"From: tree@inbuxa-migrate.test\r\n\ + To: user1@inbuxa-migrate.test\r\n\ + Subject: Tree\r\n\ + Message-ID: \r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Structural parent probe.\r\n"; + let leaves = [ + "Projects/Invoices", + "Clients/Acme/Invoices", + "Archive", + "Archive/2024", + "Archive /2024", + ]; + let structural = ["Projects", "Clients", "Clients/Acme", "Archive "]; + let listed = d + .create_leaves_without_parents(&account, &leaves, message) + .expect("create hierarchy"); + for name in structural { + let quoted = format!("\"{name}\""); + let line = listed + .iter() + .find(|l| l.ends_with("ed) || l.ends_with(&format!(" {name}"))) + .unwrap_or_else(|| panic!("{name:?} missing from LIST: {listed:?}")); + let lower = line.to_ascii_lowercase(); + assert!( + lower.contains("\\noselect") || lower.contains("\\nonexistent"), + "{name:?} should be non-selectable: {line}" + ); + } + + let archive = tmp_archive("dovecot-noselect"); + for pass in 0..2 { + let summary = sync::import_imap::run(common(&archive), dovecot_config(&account, &d.imap)) + .expect("imap import"); + assert!(!summary.any_failed(), "pass {pass}: {summary:?}"); + } + let conn = open_archive(&archive); + for path in leaves.iter().chain(&structural) { + let id = mailbox_id_by_path(&conn, path, '/') + .unwrap_or_else(|| panic!("{path:?} not in the archive tree")); + assert_eq!(mailbox_path(&conn, id, '/'), *path); + } + for path in structural { + let id = mailbox_id_by_path(&conn, path, '/').expect("structural"); + assert_eq!(emails_in_mailbox(&conn, id), 0, "{path:?}"); + } + for path in leaves { + let id = mailbox_id_by_path(&conn, path, '/').expect("leaf"); + assert_eq!(emails_in_mailbox(&conn, id), 1, "{path:?}"); + } + drop(conn); + + let fx = seeder::provision(base_url()).expect("provision"); + let dst = fx.account(seeder::SYNC_OUT[0]).expect("sync-out target"); + let export = sync::export::run( + CommonConfig { + archive: archive.clone(), + threads: 4, + dry_run: false, + max_retries: 5, + allow_invalid_certs: true, + logger: Logger::from_flags(false, 0), + }, + ExportConfig { + connect: ConnectConfig { + url: fx.base_url.clone(), + auth: Auth::Basic { + user: dst.email.clone(), + password: dst.password.clone(), + }, + account: AccountSelector::Id(dst.account_id.clone()), + }, + objects: None, + prune: false, + yes: true, + }, + ) + .expect("export"); + assert!(!export.any_failed(), "export reported failures: {export:?}"); + + let jmap = + seeder::jmap::Jmap::connect(&fx.base_url, &dst.email, &dst.password).expect("session"); + let resp = jmap + .request( + &["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + json!([["Mailbox/get", {"accountId": dst.account_id, "ids": null}, "g"]]), + ) + .expect("Mailbox/get"); + let list = resp["methodResponses"][0][1]["list"] + .as_array() + .cloned() + .unwrap_or_default(); + let by_id: HashMap<&str, (&str, Option<&str>)> = list + .iter() + .filter_map(|m| { + Some(( + m["id"].as_str()?, + (m["name"].as_str()?, m["parentId"].as_str()), + )) + }) + .collect(); + let paths: HashSet = by_id + .keys() + .map(|id| { + let mut parts = Vec::new(); + let mut cur = Some(*id); + while let Some(c) = cur { + let (name, parent) = by_id[c]; + parts.push(name); + cur = parent; + } + parts.reverse(); + parts.join("/") + }) + .collect(); + for path in leaves + .iter() + .chain(&structural) + .filter(|p| !p.contains(' ')) + { + assert!( + paths.contains(*path), + "{path:?} missing on target: {paths:?}" + ); + } + for orphan in ["Invoices", "Acme", "2024"] { + assert!( + !paths.contains(orphan), + "{orphan:?} leaked to the top level" + ); + } + + let _ = std::fs::remove_file(&archive); + seeder::teardown(&fx.base_url).expect("teardown"); +} diff --git a/tests/sync_jmap.rs b/tests/sync_jmap.rs index 3251e09..298bd38 100644 --- a/tests/sync_jmap.rs +++ b/tests/sync_jmap.rs @@ -297,8 +297,8 @@ fn seed_deep_filenode_chain(jmap: &seeder::jmap::Jmap, account_id: &str, depth: serde_json::Value::String(format!("dir-{i}")), ); obj.insert( - "nodeType".to_owned(), - serde_json::Value::String("directory".to_owned()), + "@type".to_owned(), + serde_json::Value::String("Directory".to_owned()), ); if i > 0 { obj.insert( @@ -695,20 +695,44 @@ fn prune_non_tree_type_contact_card() { const ISSUE30_CARD_UID: &str = "inbuxa-migrate-issue30-card"; const ISSUE30_EVENT_UID: &str = "inbuxa-migrate-issue30-event"; -fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec { - let uri = resource - .get(uri_key) +struct Target<'a> { + client: &'a HttpClient, + session: &'a Session, + account: &'a str, +} + +fn resource_bytes( + target: &Target<'_>, + resource: &Value, + uri_key: &str, + type_key: &str, + expect_media_type: &str, +) -> Vec { + if let Some(uri) = resource.get(uri_key).and_then(Value::as_str) { + let prefix = format!("data:{expect_media_type};base64,"); + let payload = uri + .strip_prefix(&prefix) + .unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}")); + return STANDARD.decode(payload).expect("base64 payload"); + } + let blob_id = resource + .get("blobId") .and_then(Value::as_str) - .unwrap_or_else(|| panic!("no {uri_key} on {resource}")); - assert!( - resource.get("blobId").is_none(), - "target must not hold a blobId for {uri_key}: {resource}" + .unwrap_or_else(|| panic!("neither {uri_key} nor blobId on {resource}")); + assert_eq!( + resource.get(type_key).and_then(Value::as_str), + Some(expect_media_type), + "embedded blob keeps its media type: {resource}" ); - let prefix = format!("data:{expect_media_type};base64,"); - let payload = uri - .strip_prefix(&prefix) - .unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}")); - STANDARD.decode(payload).expect("base64 payload") + inbuxa_migrate::jmap::blobxfer::download_bytes( + target.client, + target.session, + target.account, + blob_id, + expect_media_type, + "blob", + ) + .unwrap_or_else(|e| panic!("target blobId {blob_id} does not download: {e}")) } #[test] @@ -815,8 +839,13 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() { let photo_resource = card .pointer("/media/photo") .unwrap_or_else(|| panic!("card has no media.photo: {card}")); + let target = Target { + client: &client, + session: &session, + account: &tgt.account_id, + }; assert_eq!( - data_uri_bytes(photo_resource, "uri", "image/png"), + resource_bytes(&target, photo_resource, "uri", "mediaType", "image/png"), photo, "contact photo bytes survived the export: {card}" ); @@ -845,7 +874,7 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() { .and_then(|m| m.values().next()) .unwrap_or_else(|| panic!("event has no links: {event}")); assert_eq!( - data_uri_bytes(link, "href", "text/plain"), + resource_bytes(&target, link, "href", "contentType", "text/plain"), agenda, "calendar enclosure bytes survived the export: {event}" ); @@ -933,6 +962,84 @@ fn export_round_trip_and_convergence() { seeder::teardown(base_url()).expect("teardown"); } +#[test] +#[ignore = "requires Docker"] +fn export_emails_in_multiple_import_batches() { + // With maxSetObjects lowered to 2 the email export spans several Email/import batches and still converges. + let fx = seeder::provision(base_url()).expect("provision"); + let src = fx.account("test1").expect("test1"); + let tgt = fx.account("test4").expect("test4"); + let archive = tmp_archive("batches"); + + let imp = ImportConfig { + connect: ConnectConfig { + url: base_url().to_owned(), + auth: basic("test1"), + account: AccountSelector::Id(src.account_id.clone()), + }, + objects: None, + allow_source_change: false, + }; + let s = sync::import_jmap::run(common(&archive, false), imp).expect("import"); + assert!(!s.any_failed()); + let local_emails = { + let conn = Connection::open(&archive).unwrap(); + count(&conn, "emails") + }; + assert!( + local_emails > 2, + "need more than one batch of emails: {local_emails}" + ); + + let mut small = Map::new(); + small.insert("maxSetObjects".to_owned(), json!(2)); + let _guard = JmapSettingsGuard::override_settings(small); + + let probe = HttpClient::new( + basic("test4"), + RetryPolicy::new(5), + CertOverride::for_url(true, base_url()), + ); + let session = Session::discover(&probe, base_url()).expect("discover"); + let limits = session.core_limits().expect("core limits"); + assert_eq!( + limits.max_objects_in_set, 2, + "maxSetObjects override is advertised" + ); + + let e1 = sync::export::run( + common(&archive, false), + export_cfg("test4", &tgt.account_id, false), + ) + .expect("export 1"); + assert!( + !e1.per_type.iter().any(|(_, c)| c.failed > 0), + "export 1 had failures: {e1:?}" + ); + let emails = e1 + .per_type + .iter() + .find(|(t, _)| *t == "Email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!( + emails as i64, local_emails, + "every email created across batches" + ); + + let e2 = sync::export::run( + common(&archive, false), + export_cfg("test4", &tgt.account_id, false), + ) + .expect("export 2"); + let created_2: u64 = e2.per_type.iter().map(|(_, c)| c.created).sum(); + assert_eq!(created_2, 0, "second export is convergent: {e2:?}"); + + drop(_guard); + let _ = std::fs::remove_file(&archive); + seeder::teardown(base_url()).expect("teardown"); +} + #[test] #[ignore = "requires Docker"] fn export_prune_destroys_unmatched() { diff --git a/tests/sync_webdav.rs b/tests/sync_webdav.rs index 5674357..b056764 100644 --- a/tests/sync_webdav.rs +++ b/tests/sync_webdav.rs @@ -20,7 +20,7 @@ fn base_url() -> &'static str { use inbuxa_migrate::jmap::account::AccountSelector; use inbuxa_migrate::jmap::http::Auth; use inbuxa_migrate::logging::Logger; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { @@ -59,7 +59,7 @@ fn webdav_cfg(root_url: String, domain: &str, localpart: &str) -> DavImportConfi allow_cleartext: false, dav_connections: 4, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, } }