From 9d38b3a7f1d3bcf51744f49a84af82fa40a65b81 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 01:12:15 -0700 Subject: [PATCH] Add export-caldav and export-carddav Write an archive's calendars and address books into any CalDAV or CardDAV server. The home is found from the signed-in principal (the URL can be the server root), collections are matched by display name or made with MKCALENDAR / extended MKCOL, and each event or card is written with If-None-Match: * when new, or under If-Match on its etag when it differs. The comparison ignores folding, parameter order and case, PRODID, DTSTAMP and REV, and compares N and ADR on their vCard 3 parts, so a second run writes nothing on Radicale and Baikal. Refused items are counted and the run goes on; nothing is deleted. Live round trips against Radicale and Baikal, a mock test with an in-memory DAV server, and the Baikal rig's digest hashes fixed to match its test password. --- CHANGELOG.md | 5 + README.md | 5 +- docs/usage.md | 47 ++- src/cli.rs | 66 +++ src/dav/client.rs | 61 ++- src/dav/discover.rs | 37 +- src/dav/xml.rs | 57 +++ src/export_files/cards.rs | 24 +- src/main.rs | 7 + src/sync/export_dav.rs | 634 +++++++++++++++++++++++++++++ src/sync/import_dav/coordinator.rs | 2 +- src/sync/mod.rs | 1 + tests/integration/baikal.rs | 7 +- tests/integration_export_dav.rs | 216 ++++++++++ tests/mock_export_dav.rs | 415 +++++++++++++++++++ 15 files changed, 1565 insertions(+), 19 deletions(-) create mode 100644 src/sync/export_dav.rs create mode 100644 tests/integration_export_dav.rs create mode 100644 tests/mock_export_dav.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f5a8656..d181ec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a Like `export-imap`, it writes only what's missing or differs, so running it again writes nothing. A script the server refuses is counted and the rest go on; `--dry-run` checks each one with `CHECKSCRIPT`. +- `export-caldav` and `export-carddav` write an archive's calendars and + address books into any CalDAV or CardDAV server. The account's home is + found from the server's root, collections are matched by name or made, + and each event or card is written only when it's missing or differs, under + its etag, so running it again writes nothing. Nothing is deleted. ## [27.1-rc.2] diff --git a/README.md b/README.md index ebdb06c..efd9b07 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate export-files alice.sqlite out the archive as plain files inbuxa-migrate export-imap … alice.sqlite its mail into any IMAP server inbuxa-migrate export-managesieve … alice.sqlite its filters, likewise + inbuxa-migrate export-caldav … alice.sqlite its calendars into any CalDAV server + inbuxa-migrate export-carddav … alice.sqlite its contacts into any CardDAV server inbuxa-migrate export-directory … out a server's accounts, no secrets Between the two sides sits an archive: one SQLite file holding one account. @@ -39,7 +41,8 @@ and refuses a different one unless told otherwise. Export speaks JMAP only. It writes into inbuxa, or into any JMAP server that advertises the types being written. `export-imap` writes the mail into any IMAP server, `export-managesieve` the Sieve filters into any ManageSieve -server, and `export-files` is the way out to everything else: it writes +server, `export-caldav` and `export-carddav` the calendars and contacts into +any CalDAV and CardDAV server, and `export-files` is the way out to everything else: it writes the archive as a Maildir++ tree, vCard, iCalendar and Sieve files that other servers and mail, calendar and contacts apps open, with no network and no store format in the way. For a whole server, `export-directory` writes the diff --git a/docs/usage.md b/docs/usage.md index 24871aa..d0d263f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,7 +13,7 @@ and how to install it. ## Commands and global flags ``` -inbuxa-migrate [args...] +inbuxa-migrate [args...] ``` Every command takes a few global flags -- verbosity, worker pool size, retry @@ -296,7 +296,7 @@ Writes an archive's mail into an account on any IMAP server, for a target that doesn't speak JMAP. It signs in the way `import imap` does, with the same flags. Contacts, calendars and filters don't go over IMAP; write the filters with `export-managesieve`, and contacts and calendars with -`export-files`. +`export-carddav` and `export-caldav`. Like `export`, it keeps no state. Each run reads the target folder first, matching messages by Message-ID, or without one by sender, subject, date and @@ -352,6 +352,48 @@ would write with `CHECKSCRIPT`, so a script the server would refuse is counted as failed before anything is written. A server without `CHECKSCRIPT` (one that advertises no `VERSION`) gets the comparison only. +## Export to CalDAV and CardDAV + +``` +inbuxa-migrate export-caldav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dry-run] \ + + +inbuxa-migrate export-carddav …same flags… +``` + +Writes an archive's calendars into any CalDAV server, or its address books +into any CardDAV server, with the flags `import caldav` and `import carddav` +take. `--url` can be the server's root: the account's home is found from +the signed-in user's principal, through `/.well-known/caldav` or +`/.well-known/carddav` if need be, so a fresh account with nothing in it +works. Like the other exports it keeps no state, and a second run writes +nothing. + +- **Collections.** A calendar or address book is matched to the target's + by its display name. One that isn't there is made with `MKCALENDAR` or + an extended `MKCOL`, with its name, description and (for a calendar) + color, at a path made from the name that nothing in the home uses yet. +- **Items.** Each event or card goes to `.ics` or `.vcf` in its + collection; a UID that isn't a plain name is hashed instead. A new item + is written with `If-None-Match: *`. One that is there is read and + compared, and written again with `If-Match` on its etag only when it + differs. The comparison leaves out what servers rewrite on their own: + line folding, the order and case of parameters, `PRODID`, `DTSTAMP` and + `REV`. A card's `N` and `ADR` are compared on the parts vCard 3 has, + since some servers drop the rest; a change only in those newer parts is + not sent. +- An item in two collections is written to both. An item, or a + collection, the server refuses is counted as failed with the reason, and + the run goes on; a collection that can't be made has its items counted + as failed too. An item whose server gives no etag isn't overwritten. +- Nothing is deleted from the target. + +With `--dry-run` it signs in, finds the home, reads and compares what is +there, and counts what it would create and update, without writing. + ## Export to files ``` @@ -503,6 +545,7 @@ cargo test --test integration_dovecot -- --ignored --test-threads=1 cargo test --test integration_cyrus -- --ignored --test-threads=1 cargo test --test integration_export_imap -- --ignored --test-threads=1 cargo test --test integration_export_managesieve -- --ignored --test-threads=1 +cargo test --test integration_export_dav -- --ignored --test-threads=1 # Slow tests cargo test --test mock_jmap -- --ignored diff --git a/src/cli.rs b/src/cli.rs index 2ec599f..ef714eb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -21,6 +21,7 @@ use crate::jmap::account::AccountSelector; use crate::jmap::http::Auth; use crate::logging::Logger; use crate::secret; +use crate::sync::export_dav::ExportDavConfig; use crate::sync::export_imap::ExportImapConfig; use crate::sync::export_managesieve::ExportManageSieveConfig; use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; @@ -68,6 +69,16 @@ enum Command { name = "export-managesieve" )] ExportManageSieve(Box), + #[command( + about = "Write a local archive's calendars into a CalDAV server", + name = "export-caldav" + )] + ExportCaldav(Box), + #[command( + about = "Write a local archive's address books into a CardDAV server", + name = "export-carddav" + )] + ExportCarddav(Box), #[command( about = "Write a server's domains, accounts, groups, lists and aliases as JSON and CSV, without secrets", name = "export-directory" @@ -481,6 +492,40 @@ struct ExportManageSieveArgs { archive: PathBuf, } +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +struct ExportDavArgs { + #[arg(long, value_name = "URL", help = "http(s)://host[:port][/path]")] + url: String, + + #[arg(long, value_name = "USER", help = "HTTP Basic user")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: falls back to $INBUXA_MIGRATE_API_KEY or prompt)" + )] + auth_bearer: Option>, + + #[arg(long, help = "Permit credentials on http:// URLs")] + allow_cleartext: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, +} + #[derive(Args)] struct ExportFilesArgs { #[arg( @@ -605,6 +650,7 @@ pub enum Action { ExportDirectory(ExportDirectoryConfig), ExportImap(CommonConfig, ExportImapConfig), ExportManageSieve(CommonConfig, ExportManageSieveConfig), + ExportDav(CommonConfig, ExportDavConfig), Inspect(InspectConfig), } @@ -734,6 +780,8 @@ impl Cli { Command::ExportDirectory(args) => resolve_export_directory(*args), Command::ExportImap(args) => resolve_export_imap(*args), Command::ExportManageSieve(args) => resolve_export_managesieve(*args), + Command::ExportCaldav(args) => resolve_export_dav(*args, DavKindArg::Caldav), + Command::ExportCarddav(args) => resolve_export_dav(*args, DavKindArg::Carddav), Command::Inspect(args) => resolve_inspect(*args), } } @@ -1148,6 +1196,24 @@ fn resolve_export_managesieve(args: ExportManageSieveArgs) -> Result Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_dav_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + )?; + Ok(Action::ExportDav( + common, + ExportDavConfig { + kind, + url: args.url, + auth, + allow_cleartext: args.allow_cleartext, + }, + )) +} + fn resolve_export_files(args: ExportFilesArgs) -> Result { let objects = args.objects.as_deref().map(parse_object_list).transpose()?; Ok(Action::ExportFiles(ExportFilesConfig { diff --git a/src/dav/client.rs b/src/dav/client.rs index fc050bf..83efd75 100644 --- a/src/dav/client.rs +++ b/src/dav/client.rs @@ -136,6 +136,7 @@ impl DavClient { Some("application/xml; charset=utf-8"), Some(depth), ACCEPT_DAV, + None, ) } @@ -147,11 +148,54 @@ impl DavClient { Some("application/xml; charset=utf-8"), Some(depth), ACCEPT_DAV, + None, ) } pub fn get(&self, url: &str) -> Result { - self.execute("GET", url, None, None, None, ACCEPT_BINARY) + self.execute("GET", url, None, None, None, ACCEPT_BINARY, None) + } + + /// A PUT that only lands as the caller expects: `If-None-Match: *` to + /// create (never overwrite), or `If-Match` with the etag last seen. + pub fn put( + &self, + url: &str, + body: &[u8], + content_type: &str, + condition: PutCondition<'_>, + ) -> Result { + let header = match condition { + PutCondition::Create => ("If-None-Match", "*"), + PutCondition::Replace(etag) => ("If-Match", etag), + }; + self.execute( + "PUT", + url, + Some(body), + Some(content_type), + None, + ACCEPT_BINARY, + Some(header), + ) + } + + /// `MKCALENDAR` or an extended `MKCOL`, with the XML that names it. + pub fn make_collection( + &self, + method: &str, + url: &str, + body: &str, + ) -> Result { + self.execute( + method, + url, + Some(body.as_bytes()), + Some("application/xml; charset=utf-8"), + None, + ACCEPT_DAV, + None, + ) } pub fn propfind_responses( @@ -263,6 +307,7 @@ impl DavClient { } } + #[allow(clippy::too_many_arguments)] fn execute( &self, method: &str, @@ -271,6 +316,7 @@ impl DavClient { content_type: Option<&str>, depth: Option, accept: &str, + header: Option<(&str, &str)>, ) -> Result { let logger = self.logger(); let policy = self.inner.retry; @@ -289,6 +335,7 @@ impl DavClient { depth, accept, include_auth, + header, }); match outcome { Attempt::Ok { @@ -445,6 +492,7 @@ impl DavClient { depth: Some(depth), accept: ACCEPT_DAV, include_auth, + header: None, }); match send { Ok(mut resp) => { @@ -768,6 +816,7 @@ impl DavClient { depth: None, accept: ACCEPT_BINARY, include_auth, + header: None, }; let logger = self.logger(); let started = Instant::now(); @@ -831,6 +880,9 @@ impl DavClient { if let Some(ct) = req.content_type { builder = builder.header("Content-Type", ct); } + if let Some((name, value)) = req.header { + builder = builder.header(name, value); + } let payload: &[u8] = req.body.unwrap_or(&[]); let request = builder .body(payload) @@ -847,6 +899,13 @@ struct WireRequest<'a> { depth: Option, accept: &'a str, include_auth: bool, + header: Option<(&'a str, &'a str)>, +} + +#[derive(Debug, Clone, Copy)] +pub enum PutCondition<'a> { + Create, + Replace(&'a str), } pub struct DavStream { diff --git a/src/dav/discover.rs b/src/dav/discover.rs index bbc4721..45d85d2 100644 --- a/src/dav/discover.rs +++ b/src/dav/discover.rs @@ -118,6 +118,32 @@ pub fn discover( }) } +/// Where new collections go, for an export: the home set the server names +/// for the signed-in principal, even when it holds nothing yet, which on a +/// fresh account is the usual case. Only when no principal can be found is +/// a URL that already holds collections taken as the home itself. +pub fn discover_home( + client: &DavClient, + kind: DavKind, + user_url: &str, +) -> Result { + if let Some(disc) = try_via_principal_with(client, kind, user_url, true)? { + return Ok(disc); + } + if let Some(well_known) = kind.well_known_uri() { + let absolute = absolute_url(user_url, &Href::from_normalised(well_known.to_owned()))?; + if let Some(disc) = try_via_principal_with(client, kind, &absolute, true)? { + return Ok(disc); + } + } + if let Some(disc) = try_treat_url_as_home_or_collection(client, kind, user_url)? { + return Ok(disc); + } + Err(DiscoveryError::NotFound { + url: user_url.to_owned(), + }) +} + fn try_treat_url_as_home_or_collection( client: &DavClient, kind: DavKind, @@ -181,6 +207,15 @@ fn try_via_principal( client: &DavClient, kind: DavKind, url: &str, +) -> Result, DiscoveryError> { + try_via_principal_with(client, kind, url, false) +} + +fn try_via_principal_with( + client: &DavClient, + kind: DavKind, + url: &str, + allow_empty: bool, ) -> Result, DiscoveryError> { let Some(combined_body) = kind.principal_and_home_set_body() else { return Ok(None); @@ -243,7 +278,7 @@ fn try_via_principal( }; let listing_body = kind.collection_listing_body(); let collections = propfind_collections(client, kind, &home_set_url, &listing_body)?; - if collections.is_empty() { + if collections.is_empty() && !allow_empty { return Ok(None); } Ok(Some(Discovery { diff --git a/src/dav/xml.rs b/src/dav/xml.rs index 791a523..d3fe210 100644 --- a/src/dav/xml.rs +++ b/src/dav/xml.rs @@ -166,6 +166,51 @@ pub fn addressbook_multiget(hrefs: &[Href]) -> String { out } +/// The properties a new calendar is made with (RFC 4791, section 5.3.1). +pub fn mkcalendar(name: &str, description: Option<&str>, color: Option<&str>) -> String { + let mut out = String::from( + "\n\ + \ + ", + ); + push_element(&mut out, "d:displayname", name); + if let Some(description) = description { + push_element(&mut out, "c:calendar-description", description); + } + if let Some(color) = color { + push_element(&mut out, "a:calendar-color", color); + } + out.push_str(""); + out +} + +/// An extended MKCOL that makes an address book (RFC 5689, RFC 6352). +pub fn mkcol_addressbook(name: &str, description: Option<&str>) -> String { + let mut out = String::from( + "\n\ + \ + \ + ", + ); + push_element(&mut out, "d:displayname", name); + if let Some(description) = description { + push_element(&mut out, "c:addressbook-description", description); + } + out.push_str(""); + out +} + +fn push_element(out: &mut String, tag: &str, text: &str) { + out.push('<'); + out.push_str(tag); + out.push('>'); + write_escaped(out, text); + out.push_str("'); +} + fn write_href_element(out: &mut String, h: &Href) { out.push_str(""); write_escaped(out, h.as_str()); @@ -233,6 +278,18 @@ mod tests { assert!(body.contains("/dav/card/u/d/a.vcf")); } + #[test] + fn new_collections_carry_their_names_escaped() { + let body = mkcalendar("Work & ", Some("Shifts"), Some("#FF0000FF")); + assert!(body.contains("Work & <home>")); + assert!(body.contains("Shifts")); + assert!(body.contains("#FF0000FF")); + let body = mkcol_addressbook("Family", None); + assert!(body.contains("")); + assert!(body.contains("Family")); + assert!(!body.contains("addressbook-description")); + } + #[test] fn escape_handles_xml_metacharacters() { let h = Href::from_normalised("/dav/cal/u/d/<weird>.ics".to_owned()); diff --git a/src/export_files/cards.rs b/src/export_files/cards.rs index a0f52d0..0c2118d 100644 --- a/src/export_files/cards.rs +++ b/src/export_files/cards.rs @@ -32,13 +32,13 @@ pub struct Outcome { pub manifest: Value, } -struct Collection { - id: i64, - name: String, - description: Option, - color: Option, - time_zone: Option, - is_default: bool, +pub(crate) struct Collection { + pub(crate) id: i64, + pub(crate) name: String, + pub(crate) description: Option, + pub(crate) color: Option, + pub(crate) time_zone: Option, + pub(crate) is_default: bool, } pub fn write_contacts( @@ -186,7 +186,7 @@ pub fn write_calendars( }) } -fn read_collections(conn: &Connection, sql: &str) -> Result, Error> { +pub(crate) fn read_collections(conn: &Connection, sql: &str) -> Result, Error> { let mut stmt = conn.prepare(sql)?; let rows = stmt.query_map([], |row| { Ok(Collection { @@ -203,7 +203,7 @@ fn read_collections(conn: &Connection, sql: &str) -> Result, Err /// The archive keeps a card's uid beside its JSContact, and photos and /// other media as blobs; the vCard needs both back inside. -fn vcard_text(conn: &Connection, uid: &str, data: &str) -> Result { +pub(crate) fn vcard_text(conn: &Connection, uid: &str, data: &str) -> Result { let mut value: Value = serde_json::from_str(data).map_err(|e| e.to_string())?; let map = value.as_object_mut().ok_or("not a JSON object")?; map.insert("uid".to_owned(), Value::String(uid.to_owned())); @@ -216,7 +216,11 @@ fn vcard_text(conn: &Connection, uid: &str, data: &str) -> Result Result { +pub(crate) fn icalendar_text( + conn: &Connection, + data: &str, + data_type: &str, +) -> Result { let mut value: Value = serde_json::from_str(data).map_err(|e| e.to_string())?; value .as_object_mut() diff --git a/src/main.rs b/src/main.rs index 88f7c8b..ddaab74 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,6 +87,13 @@ fn run() -> i32 { let logger = config.logger; (RunOutcome::from_result(export_files::run(config)), logger) } + Action::ExportDav(common, config) => { + let logger = common.logger; + ( + RunOutcome::from_result(sync::export_dav::run(common, config)), + logger, + ) + } Action::ExportManageSieve(common, config) => { let logger = common.logger; ( diff --git a/src/sync/export_dav.rs b/src/sync/export_dav.rs new file mode 100644 index 0000000..2c910fb --- /dev/null +++ b/src/sync/export_dav.rs @@ -0,0 +1,634 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-caldav` and `export-carddav`: an archive's calendars and address +//! books written into any CalDAV or CardDAV server, converted back to +//! iCalendar and vCard as `export-files` writes them. Like `export-imap`, +//! it keeps no state. A collection is found by its display name or made; +//! each item goes to a name taken from its UID, so every run finds what the +//! last one wrote, writes what is missing or differs, and leaves the rest. +//! Nothing is deleted. + +use std::collections::{HashMap, HashSet}; + +use rusqlite::Connection; + +use crate::dav::client::{DavClient, PutCondition}; +use crate::dav::discover::{DavKind, discover_home}; +use crate::dav::href::{Href, absolute_url, normalise}; +use crate::dav::xml; +use crate::error::Error; +use crate::export_files::cards::{Collection, icalendar_text, read_collections, vcard_text}; +use crate::jmap::error::JmapError; +use crate::jmap::http::RetryPolicy; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::net::CertOverride; +use crate::sync::import_dav::coordinator::enforce_tls_policy; +use crate::sync::import_dav::{DavAuth, DavKindArg}; +use crate::sync::keys::blake3_bytes; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +pub struct ExportDavConfig { + /// `Caldav` or `Carddav`; files are not written over WebDAV. + pub kind: DavKindArg, + pub url: String, + pub auth: DavAuth, + pub allow_cleartext: bool, +} + +/// What differs between the two kinds. +struct Kind { + dav: DavKind, + collection_type: &'static str, + item_type: &'static str, + collections_sql: &'static str, + extension: &'static str, + content_type: &'static str, + make_method: &'static str, +} + +const CALDAV: Kind = Kind { + dav: DavKind::Caldav, + collection_type: "calendar", + item_type: "calendarevent", + collections_sql: "SELECT id, name, description, color, time_zone, is_default \ + FROM calendars ORDER BY sort_order, id", + extension: ".ics", + content_type: "text/calendar; charset=utf-8", + make_method: "MKCALENDAR", +}; + +const CARDDAV: Kind = Kind { + dav: DavKind::Carddav, + collection_type: "addressbook", + item_type: "contactcard", + collections_sql: "SELECT id, name, description, NULL, NULL, is_default \ + FROM address_books ORDER BY sort_order, id", + extension: ".vcf", + content_type: "text/vcard; charset=utf-8", + make_method: "MKCOL", +}; + +/// One card or event as the server will get it. +struct Item { + uid: String, + text: String, + collections: Vec, +} + +/// id, uid (cards), collection ids, data, data type (events). +type Row = (i64, Option, String, String, Option); + +fn read_items( + conn: &Connection, + kind: &Kind, + counts: &mut TypeCounts, + logger: Logger, +) -> Result, Error> { + let sql = if matches!(kind.dav, DavKind::Caldav) { + "SELECT id, NULL, calendar_ids, data, data_type FROM calendar_events ORDER BY id" + } else { + "SELECT id, uid, address_book_ids, data, NULL FROM contact_cards ORDER BY id" + }; + let mut stmt = conn.prepare(sql)?; + let rows: Vec = stmt + .query_map([], |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)) + })? + .collect::>()?; + let mut out = Vec::with_capacity(rows.len()); + for (id, uid, collections, data, data_type) in rows { + let converted = match &uid { + Some(uid) => vcard_text(conn, uid, &data), + None => icalendar_text(conn, &data, data_type.as_deref().unwrap_or("Event")), + }; + let text = match converted { + Ok(text) => text, + Err(reason) => { + logger.warn(&format!( + "{} {id}: can't be converted: {reason}", + kind.item_type + )); + counts.failed += 1; + continue; + } + }; + let Some(uid) = uid.or_else(|| first_uid(&text)) else { + logger.warn(&format!("{} {id}: has no UID; left out", kind.item_type)); + counts.failed += 1; + continue; + }; + out.push(Item { + uid, + text, + collections: serde_json::from_str(&collections).unwrap_or_default(), + }); + } + Ok(out) +} + +/// iCalendar or vCard lines with folded lines joined back up. +fn unfolded(text: &str) -> Vec { + let mut lines: Vec = Vec::new(); + for raw in text.split('\n') { + let line = raw.strip_suffix('\r').unwrap_or(raw); + if let Some(rest) = line.strip_prefix([' ', '\t']) + && let Some(last) = lines.last_mut() + { + last.push_str(rest); + continue; + } + if !line.is_empty() { + lines.push(line.to_owned()); + } + } + lines +} + +fn first_uid(text: &str) -> Option { + unfolded(text).into_iter().find_map(|l| { + let (name, value) = l.split_once(':')?; + name.eq_ignore_ascii_case("UID") + .then(|| value.trim().to_owned()) + .filter(|v| !v.is_empty()) + }) +} + +/// A content line with its name and parameter names in capitals and its +/// parameters sorted, since servers write them in their own order, and +/// without empty components at the end of its value, which some servers +/// drop (vCard 4's `N` has seven, vCard 3's five). `N` and `ADR` are +/// compared on the parts vCard 3 has: a server that keeps only those cuts +/// the rest off every time, so sending them again would change nothing. A +/// quoted parameter value keeps its `;` and `:`. +fn canonical_line(line: &str) -> String { + let mut parts: Vec = vec![String::new()]; + let mut quoted = false; + let mut value = None; + for (i, c) in line.char_indices() { + match c { + '"' => { + quoted = !quoted; + parts.last_mut().unwrap().push(c); + } + ';' if !quoted => parts.push(String::new()), + ':' if !quoted => { + value = Some(&line[i + 1..]); + break; + } + c => parts.last_mut().unwrap().push(c), + } + } + let name = parts.remove(0).to_ascii_uppercase(); + let mut params: Vec = parts + .into_iter() + .map(|p| match p.split_once('=') { + Some((k, v)) => format!("{}={v}", k.to_ascii_uppercase()), + None => p.to_ascii_uppercase(), + }) + .collect(); + params.sort(); + let mut out = name; + for p in params { + out.push(';'); + out.push_str(&p); + } + let mut value = value.unwrap_or(""); + let parts = match out.split(';').next() { + Some("N") => Some(5), + Some("ADR") => Some(7), + _ => None, + }; + if let Some(n) = parts { + value = first_components(value, n); + } + out.push(':'); + while let Some(rest) = value.strip_suffix(';') { + if rest.ends_with('\\') { + break; + } + value = rest; + } + out.push_str(value); + out +} + +/// A structured value up to its `n`th unescaped `;`. +fn first_components(value: &str, n: usize) -> &str { + let mut seen = 0; + let mut escaped = false; + for (i, c) in value.char_indices() { + match c { + '\\' if !escaped => escaped = true, + ';' if !escaped => { + seen += 1; + if seen == n { + return &value[..i]; + } + } + _ => escaped = false, + } + } + value +} + +/// The same object for this purpose: the same lines in any order, leaving +/// out the ones a server or a converter stamps on its own. +fn same_object(a: &str, b: &str) -> bool { + let comparable = |text: &str| { + let mut lines: Vec = unfolded(text) + .iter() + .map(|l| canonical_line(l)) + .filter(|l| { + let name = l.split([':', ';']).next().unwrap_or(""); + !matches!(name, "PRODID" | "DTSTAMP" | "REV") + }) + .collect(); + lines.sort(); + lines + }; + comparable(a) == comparable(b) +} + +/// A path segment for a name the source chose: kept when it is plain and +/// short, else replaced by its hash, so the same name always lands on the +/// same segment. +fn segment(name: &str) -> String { + let plain = !name.is_empty() + && name.len() <= 100 + && !name.starts_with('.') + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')); + if plain { + name.to_owned() + } else { + blake3_bytes(name.as_bytes())[..16] + .iter() + .map(|b| format!("{b:02x}")) + .collect() + } +} + +fn slug(name: &str) -> String { + let lowered: String = name + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + let trimmed = lowered.trim_matches('-'); + if trimmed.is_empty() { + segment(name) + } else { + segment(trimmed) + } +} + +/// The server's answer to one write, as this run counts it. +enum Write { + Done, + Refused(String), +} + +fn write_result(r: Result) -> Result { + match r { + Ok(_) => Ok(Write::Done), + Err(JmapError::HttpStatus { status, body }) => Ok(Write::Refused(format!( + "http {status}{}", + if body.is_empty() { + String::new() + } else { + format!(": {}", body.trim()) + } + ))), + Err(e) => Err(e.into()), + } +} + +struct Target { + url: String, + /// Each item already there, by href, with its etag. + items: HashMap>, +} + +pub fn run(common: CommonConfig, config: ExportDavConfig) -> Result { + let kind = match config.kind { + DavKindArg::Caldav => &CALDAV, + DavKindArg::Carddav => &CARDDAV, + DavKindArg::Webdav => { + return Err(Error::Usage( + "files are written with export-files, not over WebDAV".to_owned(), + )); + } + }; + if !common.archive.is_file() { + return Err(Error::Usage(format!( + "no archive at {}", + common.archive.display() + ))); + } + enforce_tls_policy(&config.url, config.allow_cleartext)?; + let conn = crate::db::init::open(&common.archive)?; + let logger = common.logger; + let dry_run = common.dry_run; + let mut collection_counts = TypeCounts::default(); + let mut item_counts = TypeCounts::default(); + let collections = read_collections(&conn, kind.collections_sql)?; + let items = read_items(&conn, kind, &mut item_counts, logger)?; + + let client = DavClient::new( + config.auth.to_jmap_auth(), + RetryPolicy::new(common.max_retries), + CertOverride::for_url(common.allow_invalid_certs, &config.url), + ); + client.set_logger(logger); + let home = discover_home(&client, kind.dav, &config.url) + .map_err(|e| Error::Connection(format!("finding where collections go: {e}")))?; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "{} home {} holds {} collection(s)", + kind.collection_type, + home.home_set_url, + home.collections.len() + ); + } + let home_url = if home.home_set_url.ends_with('/') { + home.home_set_url.clone() + } else { + format!("{}/", home.home_set_url) + }; + // Every name in the home is taken, not only this kind's collections: a + // server may keep calendars and address books side by side. + let mut taken: HashSet = client + .propfind_responses(&home_url, 1, &xml::propfind_dav_items(), &home_url)? + .responses + .into_iter() + .map(|r| r.href.as_str().trim_end_matches('/').to_owned()) + .chain( + home.collections + .iter() + .map(|c| c.href.as_str().trim_end_matches('/').to_owned()), + ) + .collect(); + let mut used: HashSet = HashSet::new(); + + for collection in &collections { + let found = home.collections.iter().enumerate().find(|(i, c)| { + !used.contains(i) && c.props.displayname.as_deref() == Some(collection.name.as_str()) + }); + let target_url = match found { + Some((i, c)) => { + used.insert(i); + collection_counts.skipped += 1; + Some(c.url.clone()) + } + None => make_collection( + &client, + kind, + collection, + &home_url, + &mut taken, + dry_run, + &mut collection_counts, + logger, + )?, + }; + let mine: Vec<&Item> = items + .iter() + .filter(|i| i.collections.contains(&collection.id)) + .collect(); + let Some(url) = target_url else { + item_counts.failed += mine.len() as u64; + continue; + }; + let target = if found.is_some() { + list_items(&client, &url)? + } else { + Target { + url, + items: HashMap::new(), + } + }; + for item in mine { + write_item( + &client, + kind, + &target, + item, + dry_run, + &mut item_counts, + logger, + )?; + } + } + + if dry_run { + logger.warn("dry run: nothing was written"); + } + Ok(Summary { + per_type: vec![ + (kind.collection_type, collection_counts), + (kind.item_type, item_counts), + ], + retries_observed: client.retries_observed(), + retry_after_sleeps: client.retry_after_sleeps(), + }) +} + +/// A new collection under the home, named after the archive's, or `None` +/// when the server won't make it. +#[allow(clippy::too_many_arguments)] +fn make_collection( + client: &DavClient, + kind: &Kind, + collection: &Collection, + home_url: &str, + taken: &mut HashSet, + dry_run: bool, + counts: &mut TypeCounts, + logger: Logger, +) -> Result, Error> { + let base = slug(&collection.name); + let mut name = base.clone(); + let mut n = 2; + let href_of = |name: &str| -> Result { + let href = normalise(home_url, name).map_err(|e| Error::Connection(e.to_string()))?; + Ok(href.as_str().trim_end_matches('/').to_owned()) + }; + while taken.contains(&href_of(&name)?) { + name = format!("{base}-{n}"); + n += 1; + } + taken.insert(href_of(&name)?); + let url = format!("{home_url}{name}/"); + let description = collection.description.as_deref().filter(|d| !d.is_empty()); + if dry_run { + counts.created += 1; + return Ok(Some(url)); + } + let body = if matches!(kind.dav, DavKind::Caldav) { + let color = collection.color.as_deref().filter(|c| !c.is_empty()); + xml::mkcalendar(&collection.name, description, color) + } else { + xml::mkcol_addressbook(&collection.name, description) + }; + match write_result(client.make_collection(kind.make_method, &url, &body))? { + Write::Done => { + counts.created += 1; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "{} {:?}: created at {url}", + kind.collection_type, collection.name + ); + } + Ok(Some(url)) + } + Write::Refused(reason) => { + logger.warn(&format!( + "{} {:?} could not be made ({reason}); its items are left out", + kind.collection_type, collection.name + )); + counts.failed += 1; + Ok(None) + } + } +} + +fn list_items(client: &DavClient, url: &str) -> Result { + let listing = client.propfind_responses(url, 1, &xml::propfind_dav_items(), url)?; + let own = normalise(url, "") + .map(Href::into_string) + .unwrap_or_default(); + let items = listing + .responses + .into_iter() + .filter(|r| !r.props.is_collection && r.href.as_str() != own) + .map(|r| (r.href.into_string(), r.props.etag)) + .collect(); + Ok(Target { + url: url.to_owned(), + items, + }) +} + +fn write_item( + client: &DavClient, + kind: &Kind, + target: &Target, + item: &Item, + dry_run: bool, + counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let name = format!("{}{}", segment(&item.uid), kind.extension); + let href = normalise(&target.url, &name).map_err(|e| Error::Connection(e.to_string()))?; + let url = absolute_url(&target.url, &href).map_err(|e| Error::Connection(e.to_string()))?; + let existing = target.items.get(href.as_str()); + let etag: Option; + let condition = match existing { + None => PutCondition::Create, + Some(listed) => { + let theirs = client.get(&url)?; + if same_object(&String::from_utf8_lossy(&theirs.body), &item.text) { + counts.skipped += 1; + return Ok(()); + } + etag = listed.clone().or(theirs.etag); + match etag.as_deref() { + Some(etag) => PutCondition::Replace(etag), + None => { + logger.warn(&format!( + "{} {}: the server gives no etag, so it isn't overwritten", + kind.item_type, item.uid + )); + counts.failed += 1; + return Ok(()); + } + } + } + }; + let replacing = matches!(condition, PutCondition::Replace(_)); + if dry_run { + if replacing { + counts.updated += 1; + } else { + counts.created += 1; + } + return Ok(()); + } + let put = client.put(&url, item.text.as_bytes(), kind.content_type, condition); + match write_result(put)? { + Write::Done if replacing => counts.updated += 1, + Write::Done => counts.created += 1, + Write::Refused(reason) => { + logger.warn(&format!( + "{} {} was refused: {reason}", + kind.item_type, item.uid + )); + counts.failed += 1; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn folded_lines_and_stamps_dont_count_as_changes() { + let ours = "BEGIN:VEVENT\r\nUID:a\r\nDTSTAMP:20260101T000000Z\r\nSUMMARY:Lunch with the\r\n team\r\nEND:VEVENT\r\n"; + let theirs = "BEGIN:VEVENT\nSUMMARY:Lunch with the team\nUID:a\nDTSTAMP:20261030T120000Z\nEND:VEVENT\n"; + assert!(same_object(ours, theirs)); + assert!(!same_object(ours, &theirs.replace("Lunch", "Dinner"))); + } + + #[test] + fn parameter_order_doesnt_count_as_a_change() { + let ours = "ATTENDEE;PARTSTAT=ACCEPTED;CN=\"Smith; Jo: PhD\":mailto:jo@example.org\r\n"; + let theirs = "attendee;cn=\"Smith; Jo: PhD\";partstat=ACCEPTED:mailto:jo@example.org\n"; + assert!(same_object(ours, theirs)); + assert_eq!( + canonical_line("ATTENDEE;PARTSTAT=ACCEPTED;CN=\"a;b:c\":mailto:x@y"), + "ATTENDEE;CN=\"a;b:c\";PARTSTAT=ACCEPTED:mailto:x@y" + ); + assert!(same_object("N:Doe;Jo;;;;;\r\n", "N:Doe;Jo;;;\n")); + assert!(!same_object("N:Doe;Jo\\;\r\n", "N:Doe;Jo\n")); + assert!(!same_object("N:Doe;Jo;;;Jr.\r\n", "N:Doe;Jo\n")); + assert!(same_object( + "ADR;TYPE=HOME:;;5 Main St;Town;OR;97477;US;;;;;5 Main St\r\n", + "ADR;TYPE=HOME:;;5 Main St;Town;OR;97477;US\n" + )); + assert!(!same_object( + "ADR:;;5 Main St;Town;OR;97477;US\r\n", + "ADR:;;6 Main St;Town;OR;97477;US\n" + )); + assert_eq!(first_components("a\\;b;c;d", 2), "a\\;b;c"); + let other = "ATTENDEE;PARTSTAT=DECLINED;CN=\"Smith; Jo: PhD\":mailto:jo@example.org\r\n"; + assert!(!same_object(ours, other)); + } + + #[test] + fn the_uid_is_read_unfolded() { + let text = + "BEGIN:VCALENDAR\r\nBEGIN:VEVENT\r\nUID:abc\r\n def@example.org\r\nEND:VEVENT\r\n"; + assert_eq!(first_uid(text).as_deref(), Some("abcdef@example.org")); + assert_eq!(first_uid("BEGIN:VEVENT\r\nEND:VEVENT\r\n"), None); + } + + #[test] + fn names_from_the_source_become_safe_segments() { + assert_eq!(segment("abc-123_x.y"), "abc-123_x.y"); + let hashed = segment("ab/../c@example.org"); + assert_eq!(hashed.len(), 32); + assert_eq!(hashed, segment("ab/../c@example.org")); + assert_eq!(segment(".hidden").len(), 32); + assert_eq!(slug("Family & Friends"), "family---friends"); + assert_eq!(slug("Équipe"), "quipe"); + assert_eq!(slug("日本").len(), 32); + assert_eq!(slug("Work"), "work"); + } +} diff --git a/src/sync/import_dav/coordinator.rs b/src/sync/import_dav/coordinator.rs index df236ca..42c0766 100644 --- a/src/sync/import_dav/coordinator.rs +++ b/src/sync/import_dav/coordinator.rs @@ -524,7 +524,7 @@ fn run_gc(conn: &Connection) -> Result<(), Error> { Ok(()) } -fn enforce_tls_policy(url: &str, allow_cleartext: bool) -> Result<(), Error> { +pub(crate) fn enforce_tls_policy(url: &str, allow_cleartext: bool) -> Result<(), Error> { let parsed = url::Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; match parsed.scheme() { diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 76d2f54..02fb65b 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -8,6 +8,7 @@ pub mod batch; pub mod emailmeta; pub mod export; +pub mod export_dav; pub mod export_imap; pub mod export_managesieve; pub mod import_dav; diff --git a/tests/integration/baikal.rs b/tests/integration/baikal.rs index 4c8188e..6066042 100644 --- a/tests/integration/baikal.rs +++ b/tests/integration/baikal.rs @@ -24,9 +24,10 @@ const HTTP_PORT: u16 = 80; const REALM: &str = "BaikalDAV"; const ADMIN_PASSWORD_HASH: &str = "004b53ae84286ba6003b34e8ef404826"; -const DIGEST_USER1: &str = "b7a47e657a4a966d0d844b1211416024"; -const DIGEST_USER2: &str = "9fc67834c5a099f0a2eccea3f6918169"; -const DIGEST_USER3: &str = "e1b30b108dad01b323a999b0bb1252ab"; +// md5(":BaikalDAV:"), as sabre/dav keeps them. +const DIGEST_USER1: &str = "e261d7451ec401d35a58d199de18d546"; +const DIGEST_USER2: &str = "138bff05db3d74685ec3b0e2d529b5b1"; +const DIGEST_USER3: &str = "e2d37cbb475478f646369e86f93f2866"; const SCHEMA: &str = include_str!("baikal_schema.sql"); diff --git a/tests/integration_export_dav.rs b/tests/integration_export_dav.rs new file mode 100644 index 0000000..bcc9090 --- /dev/null +++ b/tests/integration_export_dav.rs @@ -0,0 +1,216 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-caldav` and `export-carddav` into real Radicale and Baikal +//! servers. One account is seeded and imported; the archive is exported +//! into a second account that has nothing yet, and read back with +//! `import caldav` / `import carddav`: every collection and item arrives, +//! a second run writes nothing, and a changed event is all a third run +//! sends. + +mod integration; + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use integration::Account; +use integration::baikal::Baikal; +use integration::data::{load_icals, load_vcards}; +use integration::radicale::Radicale; +use integration::validate::{cleanup, common, open_archive, tmp_archive}; + +use inbuxa_migrate::sync::export_dav::{self, ExportDavConfig}; +use inbuxa_migrate::sync::import_dav::{self, DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::{Summary, TypeCounts}; +use rusqlite::Connection; + +fn auth(account: &Account) -> DavAuth { + DavAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + } +} + +fn import(kind: DavKindArg, url: String, account: &Account, archive: &Path) -> Summary { + let summary = import_dav::run( + common(archive), + DavImportConfig { + kind, + url, + auth: auth(account), + allow_cleartext: true, + dav_connections: 2, + multiget_batch: 25, + allow_source_change: false, + }, + ) + .expect("import"); + assert!(!summary.any_failed(), "{summary:?}"); + summary +} + +fn export(kind: DavKindArg, url: &str, account: &Account, archive: &Path) -> Summary { + export_dav::run( + common(archive), + ExportDavConfig { + kind, + url: url.to_owned(), + auth: auth(account), + allow_cleartext: true, + }, + ) + .expect("export") +} + +fn counts<'a>(summary: &'a Summary, ty: &str) -> &'a TypeCounts { + &summary.per_type.iter().find(|(t, _)| *t == ty).unwrap().1 +} + +/// Each calendar or address book by name, with its items as (uid, title). +fn contents(conn: &Connection, kind: DavKindArg) -> BTreeMap> { + let sql = match kind { + DavKindArg::Caldav => { + "SELECT c.name, coalesce(json_extract(e.data, '$.uid'), json_extract(e.data, '$.entries[0].uid')), + coalesce(json_extract(e.data, '$.title'), json_extract(e.data, '$.entries[0].title'), '') + FROM calendar_events e, json_each(e.calendar_ids) j JOIN calendars c ON c.id = j.value" + } + _ => { + "SELECT b.name, c.uid, coalesce(json_extract(c.data, '$.name.full'), '') + FROM contact_cards c, json_each(c.address_book_ids) j JOIN address_books b ON b.id = j.value" + } + }; + let mut out: BTreeMap> = BTreeMap::new(); + let mut stmt = conn.prepare(sql).unwrap(); + let rows = stmt + .query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + }) + .unwrap(); + for row in rows { + let (name, uid, title) = row.unwrap(); + out.entry(name).or_default().insert((uid, title)); + } + let empty: Vec = conn + .prepare(match kind { + DavKindArg::Caldav => "SELECT name FROM calendars", + _ => "SELECT name FROM address_books", + }) + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + for name in empty { + out.entry(name).or_default(); + } + out +} + +/// The whole round trip for one kind. `source` and `target` give each +/// account's URL; `target` is where discovery starts, not the home. +fn round_trip( + kind: DavKindArg, + label: &str, + source: (&Account, String), + target: (&Account, String), + back_url: String, +) { + let (collection_type, item_type) = match kind { + DavKindArg::Caldav => ("calendar", "calendarevent"), + _ => ("addressbook", "contactcard"), + }; + let archive = tmp_archive(&format!("export-dav-{label}-src")); + import(kind, source.1, source.0, &archive); + let expected = contents(&open_archive(&archive), kind); + assert!(!expected.is_empty(), "nothing was seeded"); + let items: u64 = expected.values().map(|s| s.len() as u64).sum(); + + let first = export(kind, &target.1, target.0, &archive); + assert!(!first.any_failed(), "{first:?}"); + assert_eq!( + counts(&first, collection_type).created, + expected.len() as u64 + ); + assert_eq!(counts(&first, item_type).created, items, "{first:?}"); + + let again = export(kind, &target.1, target.0, &archive); + let c = counts(&again, item_type); + assert_eq!( + (c.created, c.updated, c.skipped), + (0, 0, items), + "{again:?}" + ); + assert_eq!(counts(&again, collection_type).created, 0); + + let back = tmp_archive(&format!("export-dav-{label}-back")); + import(kind, back_url, target.0, &back); + assert_eq!(contents(&open_archive(&back), kind), expected); + + if matches!(kind, DavKindArg::Caldav) { + let changed = open_archive(&archive) + .execute( + "UPDATE calendar_events SET data = json_set(data, '$.title', 'Moved') + WHERE id = (SELECT min(id) FROM calendar_events WHERE json_extract(data, '$.title') IS NOT NULL)", + [], + ) + .unwrap(); + assert_eq!(changed, 1); + let third = export(kind, &target.1, target.0, &archive); + let c = counts(&third, item_type); + assert_eq!((c.created, c.updated), (0, 1), "{third:?}"); + } + + cleanup(&archive); + cleanup(&back); +} + +#[test] +#[ignore = "requires Docker"] +fn radicale_export_dav_round_trip_converges() { + let r = Radicale::start().expect("radicale start"); + let (from, to) = (&r.accounts[0], r.accounts.last().unwrap()); + assert_ne!(from.username, to.username); + r.seed_account(from, &load_icals().unwrap(), &load_vcards().unwrap()) + .expect("seed"); + for kind in [DavKindArg::Caldav, DavKindArg::Carddav] { + round_trip( + kind, + &format!("radicale-{kind:?}"), + (from, r.account_url(from)), + (to, r.base_url()), + r.account_url(to), + ); + } + r.stop().expect("radicale stop"); +} + +#[test] +#[ignore = "requires Docker"] +fn baikal_export_dav_round_trip_converges() { + let b = Baikal::start().expect("baikal start"); + let (from, to) = (&b.accounts[0], b.accounts.last().unwrap()); + assert_ne!(from.username, to.username); + b.seed_account(from, &load_icals().unwrap(), &load_vcards().unwrap()) + .expect("seed"); + for kind in [DavKindArg::Caldav, DavKindArg::Carddav] { + let home = |a: &Account| match kind { + DavKindArg::Caldav => b.calendar_home(a), + _ => b.addressbook_home(a), + }; + round_trip( + kind, + &format!("baikal-{kind:?}"), + (from, home(from)), + (to, format!("{}/", b.dav_root())), + home(to), + ); + } + b.stop().expect("baikal stop"); +} diff --git a/tests/mock_export_dav.rs b/tests/mock_export_dav.rs new file mode 100644 index 0000000..34031bc --- /dev/null +++ b/tests/mock_export_dav.rs @@ -0,0 +1,415 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `export-carddav` and `export-caldav` against a small in-memory DAV +//! server: collections are matched by name or made, a stale card is +//! replaced under its etag, a refused card is counted without stopping the +//! rest, and a second run writes nothing. + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::thread; + +use inbuxa_migrate::db::init; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::export_dav::{self, ExportDavConfig}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavKindArg, calcard}; +use inbuxa_migrate::sync::{CommonConfig, Summary, TypeCounts}; +use rusqlite::params; +use tempfile::TempDir; + +const ALICE: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-alice\r\nFN:Alice Smith\r\n\ +EMAIL:alice@example.com\r\nEND:VCARD\r\n"; +const BOB: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-bob\r\nFN:Bob Jones\r\nEND:VCARD\r\n"; +const CAROL: &str = + "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-carol\r\nFN:Carol Refused\r\nEND:VCARD\r\n"; +const EVENT: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\nBEGIN:VEVENT\r\nUID:ev-standup\r\n\ +DTSTAMP:20240101T000000Z\r\nDTSTART:20240301T090000Z\r\nDURATION:PT15M\r\n\ +SUMMARY:Standup\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; + +/// What the server holds, by path: a collection's name, or an item's body +/// and etag. +#[derive(Clone)] +enum Resource { + Collection { name: String, calendar: bool }, + Item { body: String, etag: u32 }, +} + +#[derive(Default)] +struct Store { + resources: BTreeMap, + next_etag: u32, + /// Each write as (method, path, condition header). + writes: Vec<(String, String, String)>, +} + +impl Store { + fn put_item(&mut self, path: &str, body: &str) { + self.next_etag += 1; + self.resources.insert( + path.to_owned(), + Resource::Item { + body: body.to_owned(), + etag: self.next_etag, + }, + ); + } +} + +fn props(path: &str, resource: Option<&Resource>) -> String { + let inner = match resource { + None => "".to_owned(), + Some(Resource::Collection { name, calendar }) => format!( + "{}{name}", + if *calendar { + "" + } else { + "" + } + ), + Some(Resource::Item { etag, .. }) => { + format!("\"{etag}\"") + } + }; + format!( + "{path}{inner}\ + HTTP/1.1 200 OK" + ) +} + +fn multistatus(responses: &str) -> String { + format!( + "{responses}" + ) +} + +fn displayname(body: &str) -> String { + let start = body.find("displayname>").map(|i| i + "displayname>".len()); + start + .and_then(|s| body[s..].find('<').map(|e| body[s..s + e].to_owned())) + .unwrap_or_default() +} + +/// One request against the store, as (status, etag, body). +fn handle( + store: &mut Store, + method: &str, + path: &str, + headers: &BTreeMap, + body: &str, +) -> (u16, Option, String) { + match method { + "PROPFIND" if headers.get("depth").map(String::as_str) == Some("0") => { + let home = "/p/\ + /cal/\ + /card/"; + let response = format!( + "{path}{home}\ + HTTP/1.1 200 OK" + ); + (207, None, multistatus(&response)) + } + "PROPFIND" => { + let mut out = props(path, store.resources.get(path)); + for (child, resource) in store.resources.range(path.to_owned()..) { + let Some(rest) = child.strip_prefix(path) else { + break; + }; + let direct = !rest.is_empty() && !rest.trim_end_matches('/').contains('/'); + if direct { + out.push_str(&props(child, Some(resource))); + } + } + (207, None, multistatus(&out)) + } + "MKCALENDAR" | "MKCOL" => { + store + .writes + .push((method.to_owned(), path.to_owned(), String::new())); + if store.resources.contains_key(path) { + return (405, None, String::new()); + } + store.resources.insert( + path.to_owned(), + Resource::Collection { + name: displayname(body), + calendar: method == "MKCALENDAR", + }, + ); + (201, None, String::new()) + } + "GET" => match store.resources.get(path) { + Some(Resource::Item { body, etag }) => (200, Some(*etag), body.clone()), + _ => (404, None, String::new()), + }, + "PUT" => { + let condition = headers + .get("if-none-match") + .map(|v| format!("If-None-Match: {v}")) + .or_else(|| headers.get("if-match").map(|v| format!("If-Match: {v}"))) + .unwrap_or_default(); + store + .writes + .push(("PUT".to_owned(), path.to_owned(), condition)); + if path.contains("card-carol") { + return (415, None, String::new()); + } + let current = match store.resources.get(path) { + Some(Resource::Item { etag, .. }) => Some(format!("\"{etag}\"")), + _ => None, + }; + let allowed = match (headers.get("if-none-match"), headers.get("if-match")) { + (Some(_), _) => current.is_none(), + (_, Some(want)) => current.as_deref() == Some(want.as_str()), + _ => false, + }; + if !allowed { + return (412, None, String::new()); + } + let status = if current.is_some() { 204 } else { 201 }; + store.put_item(path, body); + (status, Some(store.next_etag), String::new()) + } + _ => (405, None, String::new()), + } +} + +/// A DAV server on a local port, one request per connection. +fn serve(store: Arc>) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + continue; + } + let mut parts = line.split_whitespace(); + let method = parts.next().unwrap_or_default().to_owned(); + let path = parts.next().unwrap_or_default().to_owned(); + let mut headers = BTreeMap::new(); + loop { + let mut h = String::new(); + if reader.read_line(&mut h).unwrap_or(0) == 0 || h.trim().is_empty() { + break; + } + if let Some((k, v)) = h.split_once(':') { + headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_owned()); + } + } + let length: usize = headers + .get("content-length") + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + let body = String::from_utf8_lossy(&body); + let (status, etag, out) = + handle(&mut store.lock().unwrap(), &method, &path, &headers, &body); + let etag = etag + .map(|e| format!("ETag: \"{e}\"\r\n")) + .unwrap_or_default(); + let kind = if status == 207 { + "Content-Type: application/xml; charset=utf-8\r\n" + } else { + "" + }; + let _ = write!( + stream, + "HTTP/1.1 {status} X\r\n{etag}{kind}Content-Length: {}\r\nConnection: close\r\n\r\n{out}", + out.len() + ); + } + }); + url +} + +fn common(archive: &Path) -> CommonConfig { + CommonConfig { + archive: archive.to_path_buf(), + threads: 1, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: Logger::from_flags(false, 0), + } +} + +fn export(kind: DavKindArg, url: &str, archive: &Path) -> Summary { + export_dav::run( + common(archive), + ExportDavConfig { + kind, + url: url.to_owned(), + auth: DavAuth::Basic { + user: "u".into(), + password: "p".into(), + }, + allow_cleartext: true, + }, + ) + .expect("export") +} + +fn counts<'a>(summary: &'a Summary, ty: &str) -> &'a TypeCounts { + &summary.per_type.iter().find(|(t, _)| *t == ty).unwrap().1 +} + +fn tuple(c: &TypeCounts) -> (u64, u64, u64, u64) { + (c.created, c.updated, c.skipped, c.failed) +} + +fn seed(archive: &Path) { + let conn = init::open(archive).unwrap(); + conn.execute( + "INSERT INTO address_books (id, name, is_default) VALUES (1, 'Personal', 1), (2, 'Work', 0)", + [], + ) + .unwrap(); + for (vcard, books) in [(ALICE, "[1]"), (BOB, "[2]"), (CAROL, "[2]")] { + let card = calcard::vcard_to_jscontact(vcard, "/x").unwrap(); + conn.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![card.uid, books, card.data.to_string()], + ) + .unwrap(); + } + conn.execute( + "INSERT INTO calendars (id, name, is_default) VALUES (1, 'Team', 1)", + [], + ) + .unwrap(); + let mut entry = calcard::ical_to_jscalendar_entries(EVENT) + .unwrap() + .remove(0); + calcard::strip_extracted_fields_from_event(&mut entry.data); + conn.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES ('[1]', ?1)", + params![entry.data.to_string()], + ) + .unwrap(); +} + +#[test] +fn carddav_export_matches_by_name_replaces_under_etag_and_converges() { + let dir = TempDir::new().unwrap(); + let archive = dir.path().join("archive.sqlite"); + seed(&archive); + + let store = Arc::new(Mutex::new(Store::default())); + { + // The target already has a "Personal" book holding an older Alice. + let mut s = store.lock().unwrap(); + s.resources.insert( + "/card/contacts/".into(), + Resource::Collection { + name: "Personal".into(), + calendar: false, + }, + ); + s.put_item( + "/card/contacts/card-alice.vcf", + "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-alice\r\nFN:Alice Old\r\nEND:VCARD\r\n", + ); + } + let url = serve(store.clone()); + + let first = export(DavKindArg::Carddav, &url, &archive); + assert_eq!( + tuple(counts(&first, "addressbook")), + (1, 0, 1, 0), + "{first:?}" + ); + assert_eq!( + tuple(counts(&first, "contactcard")), + (1, 1, 0, 1), + "{first:?}" + ); + { + let s = store.lock().unwrap(); + let writes: Vec<(&str, &str, &str)> = s + .writes + .iter() + .map(|(m, p, c)| (m.as_str(), p.as_str(), c.as_str())) + .collect(); + assert_eq!( + writes, + [ + ("PUT", "/card/contacts/card-alice.vcf", "If-Match: \"1\""), + ("MKCOL", "/card/work/", ""), + ("PUT", "/card/work/card-bob.vcf", "If-None-Match: *"), + ("PUT", "/card/work/card-carol.vcf", "If-None-Match: *"), + ] + ); + let Some(Resource::Collection { name, .. }) = s.resources.get("/card/work/") else { + panic!("no Work book"); + }; + assert_eq!(name, "Work"); + let Some(Resource::Item { body, .. }) = s.resources.get("/card/contacts/card-alice.vcf") + else { + panic!("no Alice"); + }; + assert!(body.contains("Alice Smith"), "{body}"); + } + + store.lock().unwrap().writes.clear(); + let again = export(DavKindArg::Carddav, &url, &archive); + assert_eq!( + tuple(counts(&again, "addressbook")), + (0, 0, 2, 0), + "{again:?}" + ); + assert_eq!( + tuple(counts(&again, "contactcard")), + (0, 0, 2, 1), + "{again:?}" + ); + let writes = store.lock().unwrap().writes.clone(); + assert_eq!( + writes.len(), + 1, + "only the refused card is tried again: {writes:?}" + ); +} + +#[test] +fn caldav_export_makes_the_calendar_and_converges() { + let dir = TempDir::new().unwrap(); + let archive = dir.path().join("archive.sqlite"); + seed(&archive); + let store = Arc::new(Mutex::new(Store::default())); + let url = serve(store.clone()); + + let first = export(DavKindArg::Caldav, &url, &archive); + assert_eq!(tuple(counts(&first, "calendar")), (1, 0, 0, 0), "{first:?}"); + assert_eq!( + tuple(counts(&first, "calendarevent")), + (1, 0, 0, 0), + "{first:?}" + ); + { + let s = store.lock().unwrap(); + let Some(Resource::Item { body, .. }) = s.resources.get("/cal/team/ev-standup.ics") else { + panic!("no event: {:?}", s.resources.keys().collect::>()); + }; + assert!(body.contains("SUMMARY:Standup"), "{body}"); + } + + let again = export(DavKindArg::Caldav, &url, &archive); + assert_eq!(tuple(counts(&again, "calendar")), (0, 0, 1, 0), "{again:?}"); + assert_eq!( + tuple(counts(&again, "calendarevent")), + (0, 0, 1, 0), + "{again:?}" + ); +} -- 2.54.0