From 584b2b247d223bd057cd4ba8b0cd8634c6af0fe4 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 2 Nov 2026 18:24:19 -0700 Subject: [PATCH 1/2] upstream-import: leave out upstream's Dockerfile Nothing here builds it and its base image isn't pinned. The fork ships release binaries, and inbuxa-server pins those by hash. --- scripts/upstream-import.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/upstream-import.py b/scripts/upstream-import.py index d2309e0..c683da1 100755 --- a/scripts/upstream-import.py +++ b/scripts/upstream-import.py @@ -50,8 +50,9 @@ REPO_URL = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" # The fork's own copies always win: upstream's versions of these are dropped # and main's are put in their place. OWNED = ["README.md", "CHANGELOG.md", ".gitignore", ".github", ".gitea"] -# Upstream's release tooling, which the fork replaced. -DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg"] +# Upstream's release tooling, which the fork replaced, and its container build: +# the fork ships release binaries, checked by their published hashes. +DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg", "Dockerfile", ".dockerignore"] # Order matters: the specific forms before the plain name. RENAMES = [ -- 2.54.0 From a5d66c51c10d6c9032cc982daafef7e3bd80cb30 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 2 Nov 2026 18:24:20 -0700 Subject: [PATCH 2/2] Import upstream v1.0.14, renamed Upstream commit 033d2059caf2733ce6c0916b81e24b3dee54887b, renamed by scripts/upstream-import.py. --- .gitea/workflows/upstream-watch.yml | 2 +- CHANGELOG.md | 91 +++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 4 +- README.md | 6 +- src/dav/client.rs | 3 +- src/exchange_ews/client.rs | 3 +- src/exchange_graph/client.rs | 3 +- src/jmap/http.rs | 43 +++++++- src/sync/import_takeout/contacts.rs | 15 +-- src/sync/import_takeout/labels.rs | 147 +++++++++++++++++++++------- tests/sync_takeout.rs | 10 +- 12 files changed, 268 insertions(+), 61 deletions(-) diff --git a/.gitea/workflows/upstream-watch.yml b/.gitea/workflows/upstream-watch.yml index a7aca19..b563375 100644 --- a/.gitea/workflows/upstream-watch.yml +++ b/.gitea/workflows/upstream-watch.yml @@ -81,7 +81,7 @@ jobs: "git fetch origin upstream:upstream\n" f"scripts/upstream-import.py {tag}\n" f"git switch -c import/upstream-{tag} origin/main\n" - "git merge upstream\n" + "git merge refs/heads/upstream\n" "```\n\n" "Before reading the changes, scan them for hidden characters and text aimed at " "automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.") diff --git a/CHANGELOG.md b/CHANGELOG.md index c8acae8..2a8b088 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,97 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +## [27.1-rc.10] + +The tenth release candidate of the 27.1 series: real file names kept +through an account export. + +### Added +- `export-files` keeps the real name of every file, directory and link + whose name had to change on disk, in the manifest's new `files.names`, + and `import files` restores them. A real name that is taken in its + directory, or can't be a name, is left as it is on disk with a warning, + so a renamed entry never takes another's place; a listed link whose name + is taken is not made. + +## [27.1-rc.9] + +The ninth release candidate of the 27.1 series: an account export read +back whole. + +### Added +- `import files` reads an account export back into an archive: the tree + `export-files` writes, specified at + . Besides the mail, + contacts, calendars, Sieve scripts and files, it reads what only the + manifest holds: real folder names, roles and subscriptions, the default + address book and calendar, calendar colors and time zones, the active + script, media types, links and identities. Copies of a message in several + folders come back as one message in each. Paths in the manifest are + checked to stay inside the export, and no symbolic link is followed. An + archive gains the source kind `files` and a table for what it read at its + next import; until then `inspect` and the other read-only commands call it + an older layout. + +## [27.1-rc.8] + +The eighth release candidate of the 27.1 series: what a server needs to +run migrations into itself, from its admin console. + +### Added +- `migrate --progress json` writes one JSON event per line on stdout (a + step begins or ends, an account ends, the run ends), for a program that + runs a migration and shows its progress. +- `plan --answers` writes the guide's plans from its answers in a file, so + a program can ask the questions itself and run `migrate`. The answers + hold no secrets. +- `--stay-on-url` sends every request to the `--url` host, whatever the + server's session advertises, for a server reached at its own loopback or + through a tunnel. `import jmap`, `export`, `export-directory` and + `import-directory` take it. +- `import imap --google-key-from-env` reads the Google service account's + key from `$INBUXA_MIGRATE_GOOGLE_KEY`, and a plan step can name it as + the `googleKey` secret. + +## [27.1-rc.7] + +The seventh release candidate of the 27.1 series: upstream's 1.0.12 and +1.0.13 merged in, with out-of-office replies, `--replace-source`, and files +kept working with inbuxa. + +### Added +- The out-of-office reply comes across: imported from a server's + `VacationResponse`, and exported to the target's, in place of copying the + script the server keeps for it. +- WebDAV, CalDAV, CardDAV: `--replace-source` removes the other recorded + sources of the same kind, and what was imported from them, before + importing, so a moved share leaves one tree in the archive. +- Files keep their `executable` flag where the server has one. An archive + gains the column at its next import; until then `inspect` and the other + read-only commands call it an older layout. + +### Changed +- FileNodes are read in either shape servers use: `nodeType` and `type`, as + inbuxa and other 0.16-line servers spell them, or `@type` and `mediaType` + (draft-ietf-jmap-filenode-18, as in Stalwart 1.0). An export asks the + target which it takes and writes that one. Upstream dropped the older + shape; this fork keeps both. +- The Graph device-code sign-in asks for `Files.Read`, so the default run + reaches OneDrive. +- iCalendar and vCard parse failures say what was wrong and quote the line. + +### Fixed +- IMAP: children of a `\Noselect` parent folder kept their place instead of + landing at the top level. +- JMAP import no longer holds every blob of a page in memory at once, which + large messages could exhaust. +- CalDAV, CardDAV: items with bare CR line breaks, as Kerio Connect sends, + are read. +- EWS: a bare `--auth-bearer` takes the API key or token from the + environment without asking for `--ews-client-id`. +- `--account-name` as an administrator takes the data account from the + principal's `accounts`, as RFC 9670 defines it. + ## [27.1-rc.6] The sixth release candidate of the 27.1 series: whole domains from Microsoft diff --git a/Cargo.lock b/Cargo.lock index 9f34e2f..f1e76b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "inbuxa-migrate" -version = "1.0.13" +version = "1.0.14" dependencies = [ "blake3", "calcard", diff --git a/Cargo.toml b/Cargo.toml index b7de5e1..514e33e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "inbuxa-migrate" description = "Account migration and backup for inbuxa" -version = "27.1.0-rc.6" +version = "27.1.0-rc.10" authors = ["Coffey Labs LLC "] license = "Apache-2.0 OR MIT" repository = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" @@ -26,7 +26,7 @@ blake3 = "1.8.5" time = { version = "0.3.47", features = ["serde-well-known", "macros"] } url= "2.5.8" thiserror = "2.0.18" -ureq = { version = "3.3.0", default-features = false, features = ["json", "gzip", "rustls-no-provider", "platform-verifier"] } +ureq = { version = "3.5.0", default-features = false, features = ["json", "gzip", "rustls-no-provider", "platform-verifier"] } clap = { version = "4.6.1", features = ["derive"] } crossbeam-channel = "0.5.15" num_cpus = "1.17.0" diff --git a/README.md b/README.md index 3d870d6..e44db02 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate inspect alice.sqlite what landed inbuxa-migrate export … alice.sqlite the archive into inbuxa inbuxa-migrate export-files alice.sqlite out the archive as plain files + inbuxa-migrate import files out alice.sqlite those files read back, manifest and all inbuxa-migrate export-imap … alice.sqlite its mail into any IMAP server inbuxa-migrate export-managesieve … alice.sqlite its filters, likewise inbuxa-migrate export-caldav … alice.sqlite its calendars into any CalDAV server @@ -48,7 +49,8 @@ server, `export-caldav` and `export-carddav` the calendars and contacts into any CalDAV and CardDAV server, and `export-files` is the way out to everything else: it writes the archive as a Maildir++ tree, vCard, iCalendar and Sieve files that other servers and mail, calendar and contacts apps open, with no network and no -store format in the way. For a whole server, `export-directory` writes the +store format in the way, and `import files` reads such a tree back whole. +For a whole server, `export-directory` writes the list of domains, accounts, groups, lists and aliases to set up again elsewhere, without passwords, and `import-directory` sets them up on an inbuxa server, with a new password for each user. `migrate` moves many @@ -84,6 +86,8 @@ Secrets are read from the environment, or asked for at a prompt: (INBUXA_MIGRATE_TOKEN still works) INBUXA_MIGRATE_EWS_CLIENT_SECRET the OAuth client secret for app-only EWS INBUXA_MIGRATE_GRAPH_TOKEN a Microsoft Graph access token + INBUXA_MIGRATE_GOOGLE_KEY a Google service account's JSON key, + for --google-key-from-env The command line takes them too, but a secret there ends up in the shell's history and in the process list. Use the environment or the prompt. diff --git a/src/dav/client.rs b/src/dav/client.rs index 0abbbe0..666b84a 100644 --- a/src/dav/client.rs +++ b/src/dav/client.rs @@ -63,7 +63,8 @@ pub struct DavClient { impl DavClient { pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self { - let config: Config = Config::builder() + let config: Config = retry + .apply_timeouts(Config::builder()) .http_status_as_error(false) .allow_non_standard_methods(true) .max_redirects(0) diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs index f1c341b..b4a453e 100644 --- a/src/exchange_ews/client.rs +++ b/src/exchange_ews/client.rs @@ -54,7 +54,8 @@ pub struct SoapResponse { impl EwsClient { pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> EwsClient { - let config: Config = Config::builder() + let config: Config = retry + .apply_timeouts(Config::builder()) .http_status_as_error(false) .redirect_auth_headers(RedirectAuthHeaders::SameHost) .tls_config( diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs index 89b2b47..c2c9e86 100644 --- a/src/exchange_graph/client.rs +++ b/src/exchange_graph/client.rs @@ -89,7 +89,8 @@ enum Attempt { impl GraphClient { pub fn new(bearer: String, retry: RetryPolicy, allow_invalid_certs: bool) -> GraphClient { - let config: Config = Config::builder() + let config: Config = retry + .apply_timeouts(Config::builder()) .http_status_as_error(false) .redirect_auth_headers(RedirectAuthHeaders::SameHost) .tls_config( diff --git a/src/jmap/http.rs b/src/jmap/http.rs index 77c8ed8..5543cf8 100644 --- a/src/jmap/http.rs +++ b/src/jmap/http.rs @@ -12,8 +12,9 @@ use std::time::{Duration, Instant, SystemTime}; use encodify::base64::STANDARD; use serde_json::Value; use ureq::Agent; -use ureq::config::{Config, RedirectAuthHeaders}; +use ureq::config::{Config, ConfigBuilder, RedirectAuthHeaders}; use ureq::tls::{RootCerts, TlsConfig}; +use ureq::typestate::AgentScope; use ureq::{ResponseExt, http::Uri}; use crate::jmap::error::JmapError; @@ -28,6 +29,8 @@ const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); const BASIC: &str = "Basic "; +const IO_TIMEOUT: Duration = Duration::from_secs(300); + #[derive(Debug, Clone)] pub enum Auth { Basic { user: String, password: String }, @@ -55,6 +58,7 @@ pub struct RetryPolicy { pub max_retries: u32, pub base: Duration, pub cap: Duration, + pub io_timeout: Duration, } impl RetryPolicy { @@ -63,8 +67,17 @@ impl RetryPolicy { max_retries, base: Duration::from_millis(500), cap: Duration::from_secs(30), + io_timeout: IO_TIMEOUT, } } + + pub fn apply_timeouts(&self, builder: ConfigBuilder) -> ConfigBuilder { + let timeout = Some(self.io_timeout); + builder + .timeout_connect(timeout) + .timeout_per_read(timeout) + .timeout_per_write(timeout) + } } struct Inner { @@ -104,7 +117,8 @@ enum Attempt { impl HttpClient { pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self { - let config: Config = Config::builder() + let config: Config = retry + .apply_timeouts(Config::builder()) .http_status_as_error(false) .redirect_auth_headers(RedirectAuthHeaders::SameHost) .tls_config( @@ -662,6 +676,31 @@ mod tests { assert_eq!(auth.header_value(), "Bearer abc.def"); } + // A server that accepts the connection but never answers fails the request instead of hanging. + #[test] + fn silent_server_times_out() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/jmap/session", listener.local_addr().unwrap()); + let holder = std::thread::spawn(move || listener.accept().map(|(stream, _)| stream)); + let mut retry = RetryPolicy::new(0); + retry.io_timeout = Duration::from_millis(200); + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + retry, + false, + ); + let started = Instant::now(); + let err = client.get(&url).unwrap_err(); + assert!( + matches!(&err, JmapError::RetriesExhausted(m) if m.contains("timeout")), + "got {err:?}" + ); + assert!(started.elapsed() < Duration::from_secs(10)); + drop(holder.join().unwrap()); + } + #[test] fn cross_host_detects_hostname_change() { assert!(cross_host( diff --git a/src/sync/import_takeout/contacts.rs b/src/sync/import_takeout/contacts.rs index 4d13149..475261f 100644 --- a/src/sync/import_takeout/contacts.rs +++ b/src/sync/import_takeout/contacts.rs @@ -50,15 +50,7 @@ pub fn process_file( }; for (ordinal, chunk) in chunks.into_iter().enumerate() { - match insert_card( - &tx, - source_id, - book_local, - file_path, - ordinal, - &chunk, - card_counts, - ) { + match insert_card(&tx, source_id, book_local, &chunk, card_counts) { Ok(()) => {} Err(e) => { logger.warn(&format!("{file_path:?}#{ordinal}: vCard: {e}")); @@ -106,13 +98,10 @@ fn insert_card( tx: &Transaction<'_>, source_id: i64, book_local: i64, - file_path: &Path, - ordinal: usize, chunk: &str, counts: &mut TypeCounts, ) -> Result<(), Error> { - let synthetic_href = format!("{}#{ordinal}", file_path.to_string_lossy()); - let card = calcard::vcard_to_jscontact(chunk, &synthetic_href) + let card = calcard::vcard_to_jscontact(chunk, chunk) .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; let data_json = card.data.to_string(); let address_book_ids = format!("[{book_local}]"); diff --git a/src/sync/import_takeout/labels.rs b/src/sync/import_takeout/labels.rs index 90a6c57..fb9c135 100644 --- a/src/sync/import_takeout/labels.rs +++ b/src/sync/import_takeout/labels.rs @@ -69,6 +69,50 @@ fn decode_encoded_words(value: &str) -> Cow<'_, str> { } } +#[derive(Debug, Clone, Copy)] +enum SystemLabel { + Starred, + Important, + Opened, + Unread, + Category, + Mailbox(&'static str), +} + +impl SystemLabel { + fn parse(token: &str) -> Option { + hashify::map!( + token.as_bytes(), + SystemLabel, + "Starred" => SystemLabel::Starred, + "Important" => SystemLabel::Important, + "Opened" => SystemLabel::Opened, + "Unread" => SystemLabel::Unread, + "Archived" => SystemLabel::Mailbox("Archive"), + "Category Personal" => SystemLabel::Category, + "Category Promotions" => SystemLabel::Category, + "Category Social" => SystemLabel::Category, + "Category Updates" => SystemLabel::Category, + "Category Forums" => SystemLabel::Category, + "Com estrela" => SystemLabel::Starred, + "Importante" => SystemLabel::Important, + "Aberto" => SystemLabel::Opened, + "Não lidas" => SystemLabel::Unread, + "Caixa de entrada" => SystemLabel::Mailbox("Inbox"), + "Enviado" => SystemLabel::Mailbox("Sent"), + "Rascunhos" => SystemLabel::Mailbox("Drafts"), + "Lixeira" => SystemLabel::Mailbox("Trash"), + "Arquivada" => SystemLabel::Mailbox("Archive"), + "Categoria: pessoal" => SystemLabel::Category, + "Categoria: promoções" => SystemLabel::Category, + "Categoria: social" => SystemLabel::Category, + "Categoria: atualizações" => SystemLabel::Category, + "Categoria: fóruns" => SystemLabel::Category, + ) + .copied() + } +} + pub fn classify(tokens: &[String]) -> LabelClassification { let mut mailboxes: Vec = Vec::new(); let mut keywords: BTreeSet = BTreeSet::new(); @@ -76,32 +120,36 @@ pub fn classify(tokens: &[String]) -> LabelClassification { let mut had_unread = false; for tok in tokens { - if is_category_token(tok) { - continue; - } - match tok.as_str() { - "Starred" => { + let mailbox = match SystemLabel::parse(tok) { + Some(SystemLabel::Category) => continue, + Some(SystemLabel::Starred) => { keywords.insert("$flagged".to_owned()); + continue; } - "Important" => { + Some(SystemLabel::Important) => { keywords.insert("$important".to_owned()); + continue; } - "Opened" => { + Some(SystemLabel::Opened) => { had_opened = true; - keywords.insert("$seen".to_owned()); + continue; } - "Unread" => { + Some(SystemLabel::Unread) => { had_unread = true; + continue; } - other => { - let canonical = canonical_label(other); - if !mailboxes.iter().any(|m| m == &canonical) { - mailboxes.push(canonical); - } - } + Some(SystemLabel::Mailbox(name)) => name, + None => tok.as_str(), + }; + if !mailboxes.iter().any(|m| m == mailbox) { + mailboxes.push(mailbox.to_owned()); } } + if had_opened || (!had_unread && !tokens.is_empty()) { + keywords.insert("$seen".to_owned()); + } + let opened_won_over_unread = had_opened && had_unread; LabelClassification { mailboxes, @@ -144,24 +192,6 @@ pub fn assignments_for( .collect() } -fn is_category_token(t: &str) -> bool { - matches!( - t, - "Category Personal" - | "Category Promotions" - | "Category Social" - | "Category Updates" - | "Category Forums" - ) -} - -fn canonical_label(raw: &str) -> String { - match raw { - "Archived" => "Archive".to_owned(), - other => other.to_owned(), - } -} - #[cfg(test)] mod tests { use super::*; @@ -246,6 +276,57 @@ mod tests { assert!(c.had_unread_signal); } + #[test] + fn neither_opened_nor_unread_means_seen() { + let c = classify_str("Inbox,Important"); + assert!(c.keywords.contains("$seen")); + let c = classify_str("Sent"); + assert!(c.keywords.contains("$seen")); + } + + #[test] + fn no_labels_sets_no_keywords() { + assert!(classify(&[]).keywords.is_empty()); + } + + #[test] + fn pt_br_system_labels_map_like_english() { + let c = classify_str( + "Caixa de entrada,Enviado,Rascunhos,Lixeira,Arquivada,Importante,Com estrela,\ + Aberto,Categoria: pessoal,Categoria: atualizações,Categoria: promoções,Spam,Clientes", + ); + assert_eq!( + c.mailboxes, + [ + "Inbox", "Sent", "Drafts", "Trash", "Archive", "Spam", "Clientes" + ] + ); + assert_eq!(c.keywords_sorted(), ["$flagged", "$important", "$seen"]); + let roles: Vec<_> = assignments_for(&c, MappingOptions::default()) + .into_iter() + .map(|a| a.role) + .collect(); + assert_eq!( + roles, + [ + Some("inbox"), + Some("sent"), + Some("drafts"), + Some("trash"), + Some("archive"), + Some("junk"), + None + ] + ); + } + + #[test] + fn pt_br_unread_label_means_no_seen() { + let c = classify_str("Caixa de entrada,Não lidas"); + assert!(!c.keywords.contains("$seen")); + assert!(c.had_unread_signal); + } + #[test] fn opened_plus_unread_marks_collision_and_keeps_seen() { let c = classify_str("Inbox,Opened,Unread"); diff --git a/tests/sync_takeout.rs b/tests/sync_takeout.rs index d43f2a3..3b05846 100644 --- a/tests/sync_takeout.rs +++ b/tests/sync_takeout.rs @@ -258,9 +258,9 @@ fn synthetic_takeout_imports_expected_row_counts() { assert_eq!(count(&conn, "address_books"), 1); assert_eq!( count(&conn, "contact_cards"), - 6, + 4, "2 UID-bearing cards dedup across the two .vcf files (=2); \ - 2 UID-less cards get distinct synthetic UIDs per file (=4); total 6" + 2 UID-less cards dedup by content across the two files (=2); total 4" ); let by = |t: &str| { @@ -455,9 +455,9 @@ fn contacts_dedup_by_uid_across_two_vcf_files() { |r| r.get(0), ) .unwrap(); - assert!( - synthetic_uids >= 2, - "expected synthetic UIDs for the 2 UID-less cards * 2 files" + assert_eq!( + synthetic_uids, 2, + "the 2 UID-less cards appear in both files but are stored once each" ); } -- 2.54.0