From 0355fb28f792b1620ba76484ca7726fc85b3a8bd Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 08:59:12 -0700 Subject: [PATCH] Add import-directory: set up an exported directory on inbuxa Reads the directory.json export-directory writes and creates, through the registry with an administrator's credentials, the tenants, roles (then their parent roles), domains, groups, users and mailing lists, in that order, with aliases, quotas, roles, permissions and group memberships. What the server already has, by name or address, is skipped and never changed, so a second run creates nothing. Each user created gets a generated password (24 characters from 56 letters and digits without look-alikes, by rejection sampling from getrandom), written as email,password to a new mode-600 file after each batch is made. Creates go in batches within maxObjectsInSet and are sent once, never retried; refusals are counted with the server's reason and the rest go on. A dry run reads, counts and writes nothing. Tested with a stateful mock registry (export, import into an empty server, export again: same directory.json; passwords match what was stored; a second run creates nothing) and against an inbuxa 27.1-rc.4 container, where a generated password signs in. --- CHANGELOG.md | 8 + Cargo.lock | 1 + Cargo.toml | 1 + README.md | 6 +- docs/usage.md | 46 +- src/cli.rs | 96 ++++ src/export_directory/mod.rs | 16 +- src/import_directory/mod.rs | 802 ++++++++++++++++++++++++++++++ src/import_directory/passwords.rs | 114 +++++ src/lib.rs | 1 + src/main.rs | 14 +- tests/mock_import_directory.rs | 438 ++++++++++++++++ 12 files changed, 1529 insertions(+), 14 deletions(-) create mode 100644 src/import_directory/mod.rs create mode 100644 src/import_directory/passwords.rs create mode 100644 tests/mock_import_directory.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 758eaff..4028618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Added +- `import-directory` sets up on an inbuxa server what `export-directory` + wrote: tenants, roles, domains, groups, users and mailing lists, with + their aliases, quotas, roles and memberships. What the server already has + is left as it is, so running it again creates nothing. Each user created + gets a generated password, written to a new file readable by its owner + only. + ### Fixed - `inspect` opens the archive read-only. It used to create an empty archive at a mistyped path and bring an older archive's layout up to date; now diff --git a/Cargo.lock b/Cargo.lock index 416d24d..7700037 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1335,6 +1335,7 @@ dependencies = [ "crossbeam-channel", "encodify", "flate2", + "getrandom 0.3.4", "indexmap 2.14.2", "mail-parser", "mockito", diff --git a/Cargo.toml b/Cargo.toml index f1cd18c..c2d70f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ quick-xml = "0.42" chrono = { version = "0.4", default-features = false, features = ["std"] } chrono-tz = { version = "0.10", default-features = false } psl = "2.1.226" +getrandom = "0.3.4" [dev-dependencies] mockito = "1.7.2" diff --git a/README.md b/README.md index efd9b07..245904d 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate export-caldav … alice.sqlite its calendars into any CalDAV server inbuxa-migrate export-carddav … alice.sqlite its contacts into any CardDAV server inbuxa-migrate export-directory … out a server's accounts, no secrets + inbuxa-migrate import-directory … out those accounts, set up on inbuxa Between the two sides sits an archive: one SQLite file holding one account. `import` fills it from the old server; `export` writes it into the new one. @@ -47,8 +48,9 @@ the archive as a Maildir++ tree, vCard, iCalendar and Sieve files that other servers and mail, calendar and contacts apps open, with no network and no store format in the way. For a whole server, `export-directory` writes the list of domains, accounts, groups, lists and aliases to set up again -elsewhere, without passwords. The full command reference is in -[docs/usage.md](docs/usage.md). +elsewhere, without passwords, and `import-directory` sets them up on an +inbuxa server, with a new password for each user. The full command +reference is in [docs/usage.md](docs/usage.md). ## Sources diff --git a/docs/usage.md b/docs/usage.md index 241cd66..22a5f37 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,7 +13,7 @@ and how to install it. ## Commands and global flags ``` -inbuxa-migrate [args...] +inbuxa-migrate [args...] ``` Every command takes a few global flags -- verbosity, worker pool size, retry @@ -477,7 +477,8 @@ the rest of each account's data move with `import` and the exports above. Nothing secret is written, because nothing secret is asked for: every read names the fields it wants, and credentials, app passwords, OTP seeds, API keys, encryption keys and DKIM keys are not among them. Each new server -sets its own passwords and DKIM keys. +sets its own passwords and DKIM keys; `import-directory`, below, does that +on inbuxa. ``` DIR/ @@ -509,6 +510,47 @@ is made readable by its owner only. `directory.json` is written last, so a directory without one was interrupted. With `--dry-run` it reads the registry and counts what it would write, and creates nothing. +## Import a directory + +``` +inbuxa-migrate import-directory \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + --passwords [--domain ]... [--dry-run] \ + +``` + +Sets up on an inbuxa server what `export-directory` wrote: tenants, roles, +domains, groups, users and mailing lists, with their aliases, quotas, roles +and group memberships. It works through the server's registry, so the +login must be an administrator's. Each kind is made before the ones that +point at it, so a user finds its domain, its groups and its roles. + +- **What's already there stays as it is.** A domain, tenant or role with + the same name, or an account or list with the same address, is counted + as skipped and not changed. So a second run creates nothing, and a run + that stopped is finished by running it again. It also means an account + made on a run where one of its alias domains was refused keeps no alias + there: add that alias by hand. +- **Passwords.** Each user it creates gets a generated password: 24 + letters and digits, with look-alikes such as `0` and `O` left out. The + passwords go into `--passwords FILE` as `email,password` rows, written as + each batch of accounts is made. The file must not exist yet and is made + readable by its owner only. Hand the passwords out, then delete the file. + Accounts that were already there get no password and aren't listed, so + each run needs a new file. +- **DKIM.** A new domain gets inbuxa's defaults: DKIM keys are made for + it, and DNS and certificates are left to you. +- Anything the server refuses is counted as failed, with the server's + reason, and the rest are still made. An alias or custom role that isn't + on the server is left off with a warning. Creates are sent once and not + retried, so a lost reply never makes an account twice; run again to pick + up what didn't land. + +`--domain` keeps it to the domains named. With `--dry-run` it reads the +server and counts what it would create, sends nothing and writes no file; +`--passwords` isn't needed then. + ## Inspect ``` diff --git a/src/cli.rs b/src/cli.rs index ef714eb..75994cd 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -16,6 +16,7 @@ use crate::exchange_ews::types::MailboxKind; use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces}; use crate::export_directory::ExportDirectoryConfig; use crate::export_files::ExportFilesConfig; +use crate::import_directory::ImportDirectoryConfig; use crate::inspect::InspectConfig; use crate::jmap::account::AccountSelector; use crate::jmap::http::Auth; @@ -84,6 +85,11 @@ enum Command { name = "export-directory" )] ExportDirectory(Box), + #[command( + about = "Set up on an inbuxa server the domains, accounts, groups and lists export-directory wrote", + name = "import-directory" + )] + ImportDirectory(Box), #[command(about = "Show the contents of a local archive (read-only)")] Inspect(Box), } @@ -609,6 +615,75 @@ struct ExportDirectoryArgs { dir: PathBuf, } +#[derive(Args)] +struct ImportDirectoryArgs { + #[arg( + long, + value_name = "URL", + help = "The inbuxa server: base URL or full JMAP session URL" + )] + url: String, + + #[arg(long, value_name = "USER", help = "HTTP Basic user (an administrator)")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "API key or bearer token (value optional: falls back to $INBUXA_MIGRATE_API_KEY or prompt)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "FILE", + required_unless_present = "dry_run", + help = "New file for the generated passwords of the users created (mode 600)" + )] + passwords: Option, + + #[arg( + long, + value_name = "NAME", + help = "Only this domain (repeatable; default: every domain in the directory)" + )] + domain: Vec, + + #[arg(long, help = "Count what would be created; create nothing")] + dry_run: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 5, + help = "Max retry attempts per request on transient failures" + )] + max_retries: u32, + + #[arg( + long, + help = "Accept an invalid TLS certificate from the --url host only; sign-in endpoints are always verified" + )] + allow_invalid_certs: bool, + + #[arg(short = 'v', long, action = ArgAction::Count, conflicts_with = "quiet", help = "Increase log verbosity")] + verbose: u8, + + #[arg(short = 'q', long, help = "Quiet: warnings and errors only")] + quiet: bool, + + #[arg(value_name = "DIR", help = "The directory export-directory wrote")] + dir: PathBuf, +} + #[derive(Args)] struct InspectArgs { #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] @@ -648,6 +723,7 @@ pub enum Action { Export(CommonConfig, ExportConfig), ExportFiles(ExportFilesConfig), ExportDirectory(ExportDirectoryConfig), + ImportDirectory(ImportDirectoryConfig), ExportImap(CommonConfig, ExportImapConfig), ExportManageSieve(CommonConfig, ExportManageSieveConfig), ExportDav(CommonConfig, ExportDavConfig), @@ -778,6 +854,7 @@ impl Cli { Command::Export(args) => resolve_export(*args), Command::ExportFiles(args) => resolve_export_files(*args), Command::ExportDirectory(args) => resolve_export_directory(*args), + Command::ImportDirectory(args) => resolve_import_directory(*args), Command::ExportImap(args) => resolve_export_imap(*args), Command::ExportManageSieve(args) => resolve_export_managesieve(*args), Command::ExportCaldav(args) => resolve_export_dav(*args, DavKindArg::Caldav), @@ -1243,6 +1320,25 @@ fn resolve_export_directory(args: ExportDirectoryArgs) -> Result })) } +fn resolve_import_directory(args: ImportDirectoryArgs) -> Result { + let auth = resolve_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + )?; + Ok(Action::ImportDirectory(ImportDirectoryConfig { + url: args.url, + auth, + dir: args.dir, + domains: args.domain, + passwords: args.passwords, + dry_run: args.dry_run, + max_retries: args.max_retries, + allow_invalid_certs: args.allow_invalid_certs, + logger: Logger::from_flags(args.quiet, args.verbose), + })) +} + fn resolve_inspect(args: InspectArgs) -> Result { let target = match args.target.as_deref() { Some(token) => Some(ObjectType::parse(token)?), diff --git a/src/export_directory/mod.rs b/src/export_directory/mod.rs index f642c83..e86d65c 100644 --- a/src/export_directory/mod.rs +++ b/src/export_directory/mod.rs @@ -168,17 +168,17 @@ fn write(directory: &model::Directory, dir: &Path, url: &str) -> Result<(), Erro Ok(()) } -struct Reader<'a> { - client: &'a HttpClient, - api_url: &'a str, - account_id: &'a str, - urn: &'static str, - limits: Limits, +pub(crate) struct Reader<'a> { + pub(crate) client: &'a HttpClient, + pub(crate) api_url: &'a str, + pub(crate) account_id: &'a str, + pub(crate) urn: &'static str, + pub(crate) limits: Limits, } impl Reader<'_> { /// Every object of one registry type, with only `properties` (and `id`). - fn all(&self, ty: &str, properties: &[&str]) -> Result, JmapError> { + pub(crate) fn all(&self, ty: &str, properties: &[&str]) -> Result, JmapError> { let ids = self.ids(ty)?; let chunk = usize::try_from(self.limits.max_objects_in_get.max(1)).unwrap_or(500); let mut out = Vec::with_capacity(ids.len()); @@ -205,7 +205,7 @@ impl Reader<'_> { /// As `all`, for a type this login may not be allowed to read, or a /// server may not have: that is a warning and an empty list. - fn optional( + pub(crate) fn optional( &self, ty: &str, properties: &[&str], diff --git a/src/import_directory/mod.rs b/src/import_directory/mod.rs new file mode 100644 index 0000000..9186bd8 --- /dev/null +++ b/src/import_directory/mod.rs @@ -0,0 +1,802 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `import-directory`: the accounts `export-directory` wrote, set up on an +//! inbuxa server through its registry with an administrator's credentials. +//! Tenants, roles, domains, groups, users and mailing lists are created in +//! that order, so each finds what it points at. What the server already +//! has, by name or address, is left as it is, so a second run creates +//! nothing. Each user it creates gets a generated password, written to a +//! new private file for the administrator to hand out. + +pub mod passwords; + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde_json::{Map, Value, json}; + +use crate::error::Error; +use crate::export_directory::{FORMAT_VERSION, Reader}; +use crate::jmap::http::{Auth, HttpClient, RetryPolicy}; +use crate::jmap::request::{Request, URN_INBUXA_REGISTRY, check_method_error}; +use crate::jmap::session::Session; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::net::CertOverride; +use crate::sync::{Summary, TypeCounts}; + +use passwords::PasswordFile; + +pub struct ImportDirectoryConfig { + pub url: String, + pub auth: Auth, + /// The directory `export-directory` wrote. + pub dir: PathBuf, + /// Domains to set up, by name; empty for every domain in the directory. + pub domains: Vec, + /// Where the new users' passwords go; a new file. Needed unless it's a + /// dry run. + pub passwords: Option, + pub dry_run: bool, + pub max_retries: u32, + pub allow_invalid_certs: bool, + pub logger: Logger, +} + +fn str_of<'a>(v: &'a Value, key: &str) -> Option<&'a str> { + v.get(key).and_then(Value::as_str) +} + +fn strings(v: &Value, key: &str) -> Vec { + v.get(key) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +/// A registry set, `{"a": true, ...}`. +fn set(items: impl IntoIterator) -> Value { + Value::Object(items.into_iter().map(|k| (k, Value::Bool(true))).collect()) +} + +/// Copies `key` when it holds a value. +fn put(out: &mut Map, key: &str, value: Option<&Value>) { + if let Some(v) = value.filter(|v| !v.is_null()) { + out.insert(key.to_owned(), v.clone()); + } +} + +/// The directory file, checked for its format. +fn read_directory(dir: &Path) -> Result { + let path = dir.join("directory.json"); + let text = fs::read_to_string(&path).map_err(|e| { + Error::Usage(format!( + "{}: {e}; give the directory export-directory wrote", + path.display() + )) + })?; + let doc: Value = serde_json::from_str(&text) + .map_err(|e| Error::Usage(format!("{}: {e}", path.display())))?; + if str_of(&doc, "format") != Some("inbuxa-migrate-directory") { + return Err(Error::Usage(format!( + "{} is not a directory export-directory wrote", + path.display() + ))); + } + let version = doc + .get("formatVersion") + .and_then(Value::as_u64) + .unwrap_or(0); + if version != u64::from(FORMAT_VERSION) { + return Err(Error::Usage(format!( + "{} is format version {version}; this release reads version {FORMAT_VERSION}", + path.display() + ))); + } + Ok(doc) +} + +/// What the server already has, by name or lower-cased address, and what +/// this run makes, so later objects can point at both. +#[derive(Default)] +struct Known { + tenants: HashMap, + roles: HashMap, + domains: HashMap, + /// Users and groups by address. + accounts: HashMap, + lists: HashMap, +} + +impl Known { + fn read(reader: &Reader<'_>, logger: Logger) -> Result { + let mut known = Known::default(); + let named = |list: Vec, key: &str| -> HashMap { + list.iter() + .filter_map(|v| Some((str_of(v, key)?.to_owned(), str_of(v, "id")?.to_owned()))) + .collect() + }; + known.tenants = named(reader.optional("x:Tenant", &["name"], logger)?, "name"); + known.roles = named( + reader.optional("x:Role", &["description"], logger)?, + "description", + ); + let domains = reader.all("x:Domain", &["name"])?; + let domain_names: HashMap = domains + .iter() + .filter_map(|d| Some((str_of(d, "id")?.to_owned(), str_of(d, "name")?.to_owned()))) + .collect(); + for (id, name) in &domain_names { + known.domains.insert(name.to_ascii_lowercase(), id.clone()); + } + let address = |v: &Value| -> Option { + let name = str_of(v, "name")?; + if name.contains('@') { + return Some(name.to_ascii_lowercase()); + } + let domain = domain_names.get(str_of(v, "domainId")?)?; + Some(format!("{name}@{domain}").to_ascii_lowercase()) + }; + for a in reader.all("x:Account", &["name", "domainId"])? { + if let (Some(addr), Some(id)) = (address(&a), str_of(&a, "id")) { + known.accounts.insert(addr, id.to_owned()); + } + } + for l in reader.all("x:MailingList", &["name", "domainId"])? { + if let (Some(addr), Some(id)) = (address(&l), str_of(&l, "id")) { + known.lists.insert(addr, id.to_owned()); + } + } + Ok(known) + } + + fn domain_of(&self, email: &str) -> Option<&str> { + let domain = email.rsplit_once('@')?.1.to_ascii_lowercase(); + self.domains.get(&domain).map(String::as_str) + } + + fn tenant(&self, v: &Value) -> Option { + str_of(v, "tenant") + .and_then(|t| self.tenants.get(t)) + .map(|id| json!(id)) + } +} + +/// One object to create, with what to call it in the log and summary. +struct Create { + label: String, + object: Value, + /// A user's password, written once the user exists. + password: Option, +} + +/// Placeholder id for what a dry run would create. +const PLANNED: &str = "(new)"; + +struct Writer<'a> { + client: &'a HttpClient, + api_url: &'a str, + account_id: &'a str, + urn: &'static str, + chunk: usize, + dry_run: bool, + logger: Logger, +} + +impl Writer<'_> { + /// Creates each object, in batches the server takes; the id of each + /// one made, by label. A refused object is counted and the rest go on. + fn create( + &self, + ty: &str, + items: Vec, + counts: &mut TypeCounts, + passwords: &mut Option, + ) -> Result, Error> { + let mut made = HashMap::new(); + if self.dry_run { + for item in items { + counts.created += 1; + made.insert(item.label, PLANNED.to_owned()); + } + return Ok(made); + } + let mut items = items.into_iter().peekable(); + while items.peek().is_some() { + let batch: Vec = items.by_ref().take(self.chunk).collect(); + let create: Map = batch + .iter() + .enumerate() + .map(|(i, c)| (format!("c{i}"), c.object.clone())) + .collect(); + let mut req = Request::new(); + req.registry(Some(self.urn)); + req.call( + format!("{ty}/set"), + json!({ "accountId": self.account_id, "create": create }), + "s", + ); + // Sent once: a create repeated after a lost reply would make + // the object twice, where running again finds it and moves on. + let resp = req.send_once(self.client, self.api_url)?; + let mr = resp.first()?; + check_method_error(mr)?; + let created = mr.args.get("created").and_then(Value::as_object); + let refused = mr.args.get("notCreated").and_then(Value::as_object); + for (i, item) in batch.into_iter().enumerate() { + let cid = format!("c{i}"); + if let Some(id) = created + .and_then(|c| c.get(&cid)) + .and_then(|v| str_of(v, "id")) + { + counts.created += 1; + if let (Some(file), Some(pw)) = (passwords.as_mut(), &item.password) { + file.add(&item.label, pw)?; + } + if self.logger.enabled(LEVEL_PROGRESS) { + eprintln!("{ty} {}: created", item.label); + } + made.insert(item.label, id.to_owned()); + } else { + let why = refused + .and_then(|n| n.get(&cid)) + .map(describe_set_error) + .unwrap_or_else(|| "the server didn't say".to_owned()); + self.logger + .warn(&format!("{ty} {} was not created: {why}", item.label)); + counts.failed += 1; + } + } + if let Some(file) = passwords.as_mut() { + file.flush()?; + } + } + Ok(made) + } + + /// Sets properties on objects that exist, by id; failures are warnings. + fn update( + &self, + ty: &str, + updates: Map, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + if self.dry_run || updates.is_empty() { + return Ok(()); + } + let mut req = Request::new(); + req.registry(Some(self.urn)); + req.call( + format!("{ty}/set"), + json!({ "accountId": self.account_id, "update": updates }), + "u", + ); + let resp = req.send(self.client, self.api_url)?; + let mr = resp.first()?; + check_method_error(mr)?; + if let Some(refused) = mr.args.get("notUpdated").and_then(Value::as_object) { + for (id, err) in refused { + self.logger.warn(&format!( + "{ty} {id} was made but not linked: {}", + describe_set_error(err) + )); + counts.failed += 1; + } + } + Ok(()) + } +} + +fn describe_set_error(err: &Value) -> String { + let ty = str_of(err, "type").unwrap_or("error"); + match str_of(err, "description") { + Some(d) => format!("{ty}: {d}"), + None => ty.to_owned(), + } +} + +/// `{"email": ...}` aliases as the registry's list, each on a domain the +/// server has; one on a domain it lacks is left off with a warning. +fn aliases(v: &Value, known: &Known, owner: &str, logger: Logger) -> Value { + let mut out = Map::new(); + for a in v + .get("aliases") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let Some(email) = str_of(a, "email") else { + continue; + }; + let (Some((local, _)), Some(domain_id)) = (email.rsplit_once('@'), known.domain_of(email)) + else { + logger.warn(&format!( + "{owner}: alias {email} is on a domain this server doesn't have; left off" + )); + continue; + }; + if domain_id == PLANNED { + continue; + } + let mut alias = Map::new(); + alias.insert("name".into(), json!(local)); + alias.insert("domainId".into(), json!(domain_id)); + alias.insert( + "enabled".into(), + json!(a.get("enabled").and_then(Value::as_bool).unwrap_or(true)), + ); + put(&mut alias, "description", a.get("description")); + out.insert(out.len().to_string(), Value::Object(alias)); + } + Value::Object(out) +} + +fn permissions(v: &Value) -> Option { + let p = v.get("permissions").filter(|p| !p.is_null())?; + let mode = match str_of(p, "mode")? { + "merge" => "Merge", + "replace" => "Replace", + _ => return None, + }; + Some(json!({ + "@type": mode, + "enabledPermissions": set(strings(p, "enabled")), + "disabledPermissions": set(strings(p, "disabled")), + })) +} + +/// A user's (`user`, `admin`) or group's (`default`) roles, or custom +/// roles by name; a custom role the server lacks is left off. +fn roles(v: &Value, known: &Known, owner: &str, logger: Logger) -> Option { + let r = v.get("roles").filter(|r| !r.is_null())?; + if let Some(name) = r.as_str() { + let tag = match name { + "user" => "User", + "admin" => "Admin", + "default" => "Default", + other => { + logger.warn(&format!( + "{owner}: unknown role kind {other}; left to the default" + )); + return None; + } + }; + return Some(json!({ "@type": tag })); + } + let ids: Vec = strings(r, "custom") + .into_iter() + .filter_map(|name| match known.roles.get(&name) { + Some(id) if id != PLANNED => Some(id.clone()), + Some(_) => None, + None => { + logger.warn(&format!( + "{owner}: role {name} isn't on this server; left off" + )); + None + } + }) + .collect(); + Some(json!({ "@type": "Custom", "roleIds": set(ids) })) +} + +fn sub_addressing(v: &Value) -> Option { + match v.get("subAddressing")? { + Value::String(s) if s == "enabled" => Some(json!({ "@type": "Enabled" })), + Value::String(s) if s == "disabled" => Some(json!({ "@type": "Disabled" })), + Value::Object(o) => Some(json!({ + "@type": "Custom", + "customRule": o.get("customRule").cloned().unwrap_or(Value::Null), + })), + _ => None, + } +} + +/// An account (`User` or `Group`) as the registry creates it. +fn account( + a: &Value, + ty: &str, + known: &Known, + logger: Logger, +) -> Option<(String, Map)> { + let email = str_of(a, "email")?.to_owned(); + let domain_id = known.domain_of(&email)?.to_owned(); + let name = str_of(a, "name") + .map(str::to_owned) + .or_else(|| email.rsplit_once('@').map(|(l, _)| l.to_owned()))?; + let mut o = Map::new(); + o.insert("@type".into(), json!(ty)); + o.insert("name".into(), json!(name)); + o.insert("domainId".into(), json!(domain_id)); + put(&mut o, "description", a.get("description")); + put(&mut o, "memberTenantId", known.tenant(a).as_ref()); + put(&mut o, "roles", roles(a, known, &email, logger).as_ref()); + put(&mut o, "permissions", permissions(a).as_ref()); + put(&mut o, "quotas", a.get("quotas")); + o.insert("aliases".into(), aliases(a, known, &email, logger)); + put(&mut o, "locale", a.get("locale")); + put(&mut o, "timeZone", a.get("timeZone")); + Some((email, o)) +} + +pub fn run(config: ImportDirectoryConfig) -> Result { + let logger = config.logger; + let doc = read_directory(&config.dir)?; + let only: Vec = config + .domains + .iter() + .map(|d| d.to_ascii_lowercase()) + .collect(); + let in_scope = |domain: Option<&str>| -> bool { + only.is_empty() || domain.is_some_and(|d| only.contains(&d.to_ascii_lowercase())) + }; + let list = |key: &str| -> Vec { + doc.get(key) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default() + }; + let domains: Vec = list("domains") + .into_iter() + .filter(|d| in_scope(str_of(d, "name"))) + .collect(); + for wanted in &only { + if !domains + .iter() + .any(|d| str_of(d, "name").is_some_and(|n| n.eq_ignore_ascii_case(wanted))) + { + return Err(Error::Usage(format!( + "the directory has no domain {wanted}" + ))); + } + } + let of_domain = |v: &Value| in_scope(str_of(v, "domain")); + let users: Vec = list("users").into_iter().filter(of_domain).collect(); + let groups: Vec = list("groups").into_iter().filter(of_domain).collect(); + let lists: Vec = list("lists").into_iter().filter(of_domain).collect(); + + let mut passwords = match (&config.passwords, config.dry_run) { + (_, true) => None, + (Some(path), false) => Some(PasswordFile::create(path)?), + (None, false) => { + return Err(Error::Usage( + "--passwords FILE is needed: each user created gets a generated password, \ + written there" + .to_owned(), + )); + } + }; + + let client = HttpClient::new( + config.auth.clone(), + RetryPolicy::new(config.max_retries), + CertOverride::for_url(config.allow_invalid_certs, &config.url), + ); + client.set_logger(logger); + let session = Session::discover(&client, &config.url)?; + for mismatch in session.origin_mismatches(&config.url) { + logger.warn(&mismatch.to_string()); + } + let limits = session.core_limits()?; + client.set_limits(&limits); + let (account_id, urn) = session.registry_account().ok_or_else(|| { + Error::Account(format!( + "this login can't change the server's registry ({URN_INBUXA_REGISTRY}); \ + import-directory needs an administrator's credentials" + )) + })?; + let chunk = usize::try_from(limits.max_objects_in_set.clamp(1, 500)).unwrap_or(500); + let reader = Reader { + client: &client, + api_url: &session.api_url, + account_id, + urn, + limits, + }; + let mut known = Known::read(&reader, logger)?; + let writer = Writer { + client: &client, + api_url: &session.api_url, + account_id, + urn, + chunk, + dry_run: config.dry_run, + logger, + }; + let mut counts: HashMap<&'static str, TypeCounts> = HashMap::new(); + + // Tenants. + let mut todo = Vec::new(); + let c = counts.entry("tenant").or_default(); + for t in list("tenants") { + let Some(name) = str_of(&t, "name") else { + continue; + }; + if known.tenants.contains_key(name) { + c.skipped += 1; + continue; + } + let mut o = Map::new(); + o.insert("name".into(), json!(name)); + put(&mut o, "quotas", t.get("quotas")); + todo.push(Create { + label: name.to_owned(), + object: Value::Object(o), + password: None, + }); + } + known + .tenants + .extend(writer.create("x:Tenant", todo, c, &mut passwords)?); + + // Roles: made first, then pointed at the roles they build on, which + // may be made in the same pass. + let roles_in = list("roles"); + let mut todo = Vec::new(); + let c = counts.entry("role").or_default(); + for r in &roles_in { + let Some(name) = str_of(r, "name") else { + continue; + }; + if known.roles.contains_key(name) { + c.skipped += 1; + continue; + } + let mut o = Map::new(); + o.insert("description".into(), json!(name)); + put(&mut o, "memberTenantId", known.tenant(r).as_ref()); + o.insert( + "enabledPermissions".into(), + set(strings(r, "enabledPermissions")), + ); + o.insert( + "disabledPermissions".into(), + set(strings(r, "disabledPermissions")), + ); + todo.push(Create { + label: name.to_owned(), + object: Value::Object(o), + password: None, + }); + } + let made_roles = writer.create("x:Role", todo, c, &mut passwords)?; + known.roles.extend(made_roles.clone()); + let mut links = Map::new(); + for r in &roles_in { + let Some(id) = str_of(r, "name").and_then(|n| made_roles.get(n)) else { + continue; + }; + let parents: Vec = strings(r, "roles") + .iter() + .filter_map(|n| known.roles.get(n).cloned()) + .collect(); + if !parents.is_empty() { + links.insert(id.clone(), json!({ "roleIds": set(parents) })); + } + } + writer.update("x:Role", links, c)?; + + // Domains. + let mut todo = Vec::new(); + let c = counts.entry("domain").or_default(); + for d in &domains { + let Some(name) = str_of(d, "name") else { + continue; + }; + if known.domains.contains_key(&name.to_ascii_lowercase()) { + c.skipped += 1; + continue; + } + let mut o = Map::new(); + o.insert("name".into(), json!(name)); + o.insert( + "isEnabled".into(), + json!(d.get("enabled").and_then(Value::as_bool).unwrap_or(true)), + ); + put(&mut o, "description", d.get("description")); + o.insert("aliases".into(), set(strings(d, "aliases"))); + put(&mut o, "catchAllAddress", d.get("catchAllAddress")); + put(&mut o, "subAddressing", sub_addressing(d).as_ref()); + put(&mut o, "memberTenantId", known.tenant(d).as_ref()); + todo.push(Create { + label: name.to_owned(), + object: Value::Object(o), + password: None, + }); + } + for (name, id) in writer.create("x:Domain", todo, c, &mut passwords)? { + known.domains.insert(name.to_ascii_lowercase(), id); + } + + // Groups, then the users in them. + for (kind, ty, entries) in [("group", "Group", &groups), ("user", "User", &users)] { + let mut todo = Vec::new(); + let c = counts.entry(kind).or_default(); + for a in entries { + let Some(email) = str_of(a, "email") else { + continue; + }; + if known.accounts.contains_key(&email.to_ascii_lowercase()) { + c.skipped += 1; + continue; + } + let Some((email, mut o)) = account(a, ty, &known, logger) else { + logger.warn(&format!( + "{kind} {email}: its domain isn't on this server; left out" + )); + c.failed += 1; + continue; + }; + if config.dry_run && known.domain_of(&email) == Some(PLANNED) { + c.created += 1; + continue; + } + let mut password = None; + if ty == "User" { + let groups: Vec = strings(a, "groups") + .iter() + .filter_map(|g| known.accounts.get(&g.to_ascii_lowercase()).cloned()) + .filter(|id| id != PLANNED) + .collect(); + o.insert("memberGroupIds".into(), set(groups)); + put(&mut o, "externalId", a.get("externalId")); + let pw = passwords::generate()?; + o.insert( + "credentials".into(), + json!({ "0": { "@type": "Password", "secret": pw } }), + ); + password = Some(pw); + } + todo.push(Create { + label: email, + object: Value::Object(o), + password, + }); + } + for (email, id) in writer.create("x:Account", todo, c, &mut passwords)? { + known.accounts.insert(email.to_ascii_lowercase(), id); + } + } + + // Mailing lists. + let mut todo = Vec::new(); + let c = counts.entry("mailinglist").or_default(); + for l in &lists { + let Some(email) = str_of(l, "email") else { + continue; + }; + if known.lists.contains_key(&email.to_ascii_lowercase()) { + c.skipped += 1; + continue; + } + let Some(domain_id) = known.domain_of(email).map(str::to_owned) else { + logger.warn(&format!( + "mailing list {email}: its domain isn't on this server; left out" + )); + c.failed += 1; + continue; + }; + if domain_id == PLANNED { + c.created += 1; + continue; + } + let mut o = Map::new(); + let name = str_of(l, "name") + .map(str::to_owned) + .or_else(|| email.rsplit_once('@').map(|(n, _)| n.to_owned())) + .unwrap_or_default(); + o.insert("name".into(), json!(name)); + o.insert("domainId".into(), json!(domain_id)); + put(&mut o, "description", l.get("description")); + o.insert("recipients".into(), set(strings(l, "recipients"))); + o.insert("aliases".into(), aliases(l, &known, email, logger)); + put(&mut o, "memberTenantId", known.tenant(l).as_ref()); + todo.push(Create { + label: email.to_owned(), + object: Value::Object(o), + password: None, + }); + } + writer.create("x:MailingList", todo, c, &mut passwords)?; + + if config.dry_run { + logger.warn("dry run: nothing was created and no passwords were written"); + } else if let Some(path) = &config.passwords { + let made = counts.get("user").map_or(0, |c| c.created); + logger.warn(&format!( + "{made} new user password(s) are in {}; hand them out, then delete the file", + path.display() + )); + } + let mut summary = Summary::default(); + for ty in ["tenant", "role", "domain", "group", "user", "mailinglist"] { + summary + .per_type + .push((ty, counts.remove(ty).unwrap_or_default())); + } + Ok(summary) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn known() -> Known { + let mut k = Known::default(); + k.domains.insert("example.org".into(), "d1".into()); + k.roles.insert("Helpdesk".into(), "r1".into()); + k + } + + #[test] + fn aliases_go_on_domains_the_server_has() { + let v = json!({"aliases": [ + {"email": "info@example.org", "enabled": false, "description": "front desk"}, + {"email": "x@elsewhere.net"} + ]}); + assert_eq!( + aliases(&v, &known(), "ann@example.org", Logger::from_flags(true, 0)), + json!({"0": {"name": "info", "domainId": "d1", "enabled": false, + "description": "front desk"}}) + ); + } + + #[test] + fn roles_and_permissions_take_the_registry_shape() { + let log = Logger::from_flags(true, 0); + let k = known(); + assert_eq!( + roles(&json!({"roles": "admin"}), &k, "a", log), + Some(json!({"@type": "Admin"})) + ); + assert_eq!( + roles(&json!({"roles": "default"}), &k, "a", log), + Some(json!({"@type": "Default"})) + ); + assert_eq!( + roles( + &json!({"roles": {"custom": ["Helpdesk", "Gone"]}}), + &k, + "a", + log + ), + Some(json!({"@type": "Custom", "roleIds": {"r1": true}})) + ); + assert_eq!(roles(&json!({"roles": null}), &k, "a", log), None); + assert_eq!( + permissions( + &json!({"permissions": {"mode": "merge", "enabled": ["a"], "disabled": []}}) + ), + Some(json!({"@type": "Merge", "enabledPermissions": {"a": true}, + "disabledPermissions": {}})) + ); + assert_eq!( + sub_addressing(&json!({"subAddressing": {"customRule": "r"}})), + Some(json!({"@type": "Custom", "customRule": "r"})) + ); + } + + #[test] + fn only_a_directory_export_directory_wrote_is_read() { + let dir = tempfile::TempDir::new().unwrap(); + assert!(read_directory(dir.path()).is_err()); + fs::write(dir.path().join("directory.json"), r#"{"format":"other"}"#).unwrap(); + let err = read_directory(dir.path()).unwrap_err().to_string(); + assert!(err.contains("not a directory"), "{err}"); + fs::write( + dir.path().join("directory.json"), + r#"{"format":"inbuxa-migrate-directory","formatVersion":99}"#, + ) + .unwrap(); + let err = read_directory(dir.path()).unwrap_err().to_string(); + assert!(err.contains("version 99"), "{err}"); + } +} diff --git a/src/import_directory/passwords.rs b/src/import_directory/passwords.rs new file mode 100644 index 0000000..c540d0d --- /dev/null +++ b/src/import_directory/passwords.rs @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Passwords for the accounts `import-directory` creates, and the private +//! file they are handed to the administrator in. + +use std::fs::File; +use std::io::Write; +use std::path::Path; + +use crate::error::Error; + +/// Letters and digits with the look-alikes (`0 O o 1 l I`) left out, so a +/// password can be read aloud or typed from paper. No character a +/// spreadsheet would take for a formula. +const ALPHABET: &[u8] = b"ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz23456789"; + +/// 24 characters from 56: about 139 bits, far past any strength check. +const LENGTH: usize = 24; + +pub fn generate() -> Result { + let mut out = String::with_capacity(LENGTH); + // The largest multiple of the alphabet's size below 256, so every + // character is equally likely. + let limit = 256 - 256 % ALPHABET.len(); + let mut buf = [0u8; 64]; + while out.len() < LENGTH { + getrandom::fill(&mut buf) + .map_err(|e| Error::Partial(format!("no random numbers for a password: {e}")))?; + for &b in &buf { + if (b as usize) < limit && out.len() < LENGTH { + out.push(ALPHABET[b as usize % ALPHABET.len()] as char); + } + } + } + Ok(out) +} + +/// `email,password` rows, one per account created, in a file that is new +/// and readable by its owner only. Rows are written as each batch of +/// accounts is made, so a run that stops part way keeps the passwords of +/// the accounts it made. +pub struct PasswordFile { + file: File, +} + +impl PasswordFile { + pub fn create(path: &Path) -> Result { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path).map_err(|e| match e.kind() { + std::io::ErrorKind::AlreadyExists => Error::Usage(format!( + "{} already exists; passwords go into a new file", + path.display() + )), + _ => Error::Io(e), + })?; + file.write_all(b"email,password\n")?; + Ok(PasswordFile { file }) + } + + pub fn add(&mut self, email: &str, password: &str) -> Result<(), Error> { + // Addresses hold no comma or quote, and passwords are letters and + // digits, so no field needs quoting. + writeln!(self.file, "{email},{password}")?; + Ok(()) + } + + pub fn flush(&mut self) -> Result<(), Error> { + self.file.sync_all()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn passwords_are_long_varied_and_from_the_alphabet() { + let a = generate().unwrap(); + let b = generate().unwrap(); + assert_eq!(a.len(), LENGTH); + assert_ne!(a, b); + assert!(a.bytes().all(|c| ALPHABET.contains(&c)), "{a}"); + } + + #[cfg(unix)] + #[test] + fn the_file_is_new_and_private() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("passwords.csv"); + let mut f = PasswordFile::create(&path).unwrap(); + f.add("ann@example.org", "Secret23").unwrap(); + f.flush().unwrap(); + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "email,password\nann@example.org,Secret23\n" + ); + let err = PasswordFile::create(&path).err().unwrap().to_string(); + assert!(err.contains("already exists"), "{err}"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 199866a..62b2732 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod exchange_graph; pub mod export_directory; pub mod export_files; pub mod imap; +pub mod import_directory; pub mod inspect; pub mod jmap; pub mod logging; diff --git a/src/main.rs b/src/main.rs index ddaab74..e654fb0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ use clap::Parser; use inbuxa_migrate::cli::{Action, Cli}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::{self, RunOutcome, Summary}; -use inbuxa_migrate::{export_directory, export_files, inspect}; +use inbuxa_migrate::{export_directory, export_files, import_directory, inspect}; fn main() { let code = run(); @@ -32,6 +32,7 @@ fn run() -> i32 { }; let mut quiet_report = false; + let mut partial = "some objects failed; the archive is consistent and resumable"; let (outcome, logger) = match action { Action::Import(common, config) => { let logger = common.logger; @@ -108,6 +109,15 @@ fn run() -> i32 { logger, ) } + Action::ImportDirectory(config) => { + let logger = config.logger; + partial = "some objects were not created; fix what the warnings name and run \ + again, with a new --passwords file"; + ( + RunOutcome::from_result(import_directory::run(config)), + logger, + ) + } Action::ExportImap(common, config) => { let logger = common.logger; ( @@ -130,7 +140,7 @@ fn run() -> i32 { Some(err) => fail(&err), None => { if outcome.summary.any_failed() { - logger.error("some objects failed; the archive is consistent and resumable"); + logger.error(partial); 5 } else { 0 diff --git a/tests/mock_import_directory.rs b/tests/mock_import_directory.rs new file mode 100644 index 0000000..62034a1 --- /dev/null +++ b/tests/mock_import_directory.rs @@ -0,0 +1,438 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `import-directory` into a registry served by a stateful mock: a +//! directory exported from one server, imported into an empty one and +//! exported again, comes out the same; every new user has the password +//! written for it; a second run creates nothing; a dry run sends no set. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use inbuxa_migrate::export_directory::{self, ExportDirectoryConfig}; +use inbuxa_migrate::import_directory::{self, ImportDirectoryConfig}; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::{Summary, TypeCounts}; +use serde_json::{Map, Value, json}; +use tempfile::TempDir; + +/// Objects by type, each with its id; and how many /set calls came in. +#[derive(Default)] +struct Store { + objects: BTreeMap>, + next_id: u64, + sets: u64, +} + +impl Store { + fn get(&self, ty: &str, id: &str) -> Option<&Value> { + self.objects.get(ty)?.iter().find(|o| o["id"] == id) + } + + /// The registry's own checks that matter here: what an object points + /// at must exist, names are unique, and a user's password comes in its + /// credentials. + fn check(&self, ty: &str, o: &Value) -> Result<(), String> { + let list = self.objects.get(ty).map(Vec::as_slice).unwrap_or_default(); + let key = |v: &Value| match ty { + "x:Role" => v["description"].clone(), + "x:Account" | "x:MailingList" => json!([v["name"], v["domainId"]]), + _ => v["name"].clone(), + }; + if list.iter().any(|v| key(v) == key(o)) { + return Err("alreadyExists".into()); + } + if let Some(d) = o.get("domainId") + && self + .get("x:Domain", d.as_str().unwrap_or_default()) + .is_none() + { + return Err(format!("no domain {d}")); + } + if let Some(id) = o.get("memberTenantId").and_then(Value::as_str) + && self.get("x:Tenant", id).is_none() + { + return Err(format!("no tenant {id}")); + } + if o["@type"] == "User" { + let secret = o["credentials"]["0"]["secret"].as_str().unwrap_or_default(); + if o["credentials"]["0"]["@type"] != "Password" || secret.len() < 8 { + return Err("no usable password".into()); + } + } + if o["name"] == "refused" { + return Err("invalidProperties".into()); + } + Ok(()) + } +} + +fn session(base: &str) -> String { + json!({ + "apiUrl": format!("{base}/jmap/api"), + "uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"), + "downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"), + "capabilities": { + "urn:ietf:params:jmap:core": { + "maxObjectsInGet": 500, "maxObjectsInSet": 2, "maxCallsInRequest": 16, + "maxConcurrentRequests": 4, "maxConcurrentUpload": 4, + "maxSizeRequest": 10000000, "maxSizeUpload": 50000000 + } + }, + "accounts": { + "adm": { "name": "admin", "accountCapabilities": { "urn:inbuxa:jmap:registry": {} } } + } + }) + .to_string() +} + +struct Registry { + server: mockito::ServerGuard, + store: Arc>, +} + +impl Registry { + fn url(&self) -> String { + format!("{}/jmap/session", self.server.url()) + } +} + +fn registry(store: Store) -> Registry { + let mut server = mockito::Server::new(); + let base = server.url(); + server + .mock("GET", "/jmap/session") + .with_status(200) + .with_body(session(&base)) + .create(); + let store = Arc::new(Mutex::new(store)); + let shared = store.clone(); + server + .mock("POST", "/jmap/api") + .with_status(200) + .with_body_from_request(move |req| { + let body: Value = serde_json::from_slice(req.body().unwrap()).unwrap(); + let call = &body["methodCalls"][0]; + let (method, args) = (call[0].as_str().unwrap(), &call[1]); + assert_eq!(args["accountId"], "adm"); + let (ty, verb) = method.split_once('/').unwrap(); + let mut s = shared.lock().unwrap(); + let response = match verb { + "query" => { + let from = args["position"].as_u64().unwrap_or(0) as usize; + let ids: Vec = s + .objects + .get(ty) + .into_iter() + .flatten() + .skip(from) + .map(|o| o["id"].clone()) + .collect(); + json!({"ids": ids, "position": from}) + } + "get" => { + let props: Vec<&str> = args["properties"] + .as_array() + .unwrap() + .iter() + .map(|p| p.as_str().unwrap()) + .collect(); + let ids = args["ids"].as_array().unwrap(); + let list: Vec = s + .objects + .get(ty) + .into_iter() + .flatten() + .filter(|o| ids.contains(&o["id"])) + .map(|o| { + let kept: Map = o + .as_object() + .unwrap() + .iter() + .filter(|(k, _)| *k == "id" || props.contains(&k.as_str())) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + Value::Object(kept) + }) + .collect(); + json!({"list": list, "notFound": []}) + } + "set" => { + s.sets += 1; + let mut created = Map::new(); + let mut not_created = Map::new(); + let create = args["create"].as_object().cloned().unwrap_or_default(); + assert!(create.len() <= 2, "batches keep to maxObjectsInSet"); + for (cid, o) in create { + match s.check(ty, &o) { + Ok(()) => { + s.next_id += 1; + let id = format!("n{}", s.next_id); + let mut o = o.as_object().unwrap().clone(); + o.insert("id".into(), json!(id)); + s.objects + .entry(ty.to_owned()) + .or_default() + .push(Value::Object(o)); + created.insert(cid, json!({"id": id})); + } + Err(why) => { + not_created.insert( + cid, + json!({"type": "invalidProperties", "description": why}), + ); + } + } + } + let update = args["update"].as_object().cloned().unwrap_or_default(); + for (id, patch) in update { + let list = s.objects.get_mut(ty).unwrap(); + let o = list.iter_mut().find(|o| o["id"] == id.as_str()).unwrap(); + for (k, v) in patch.as_object().unwrap() { + o[k] = v.clone(); + } + } + json!({"created": created, "notCreated": not_created}) + } + other => panic!("unexpected {other}"), + }; + json!({"methodResponses": [[method, response, call[2]]]}) + .to_string() + .into_bytes() + }) + .create(); + Registry { server, store } +} + +/// A source server with one of everything, linked up. +fn source() -> Store { + let mut s = Store::default(); + let mut add = |ty: &str, v: Value| s.objects.entry(ty.to_owned()).or_default().push(v); + add( + "x:Tenant", + json!({"id": "t", "name": "Acme", "quotas": {"maxAccounts": 50}}), + ); + add( + "x:Role", + json!({"id": "r1", "description": "Helpdesk", "memberTenantId": "t", + "roleIds": {}, "enabledPermissions": {"impersonate": true}, "disabledPermissions": {}}), + ); + add( + "x:Role", + json!({"id": "r2", "description": "Helpdesk lead", "roleIds": {"r1": true}, + "enabledPermissions": {}, "disabledPermissions": {"deleteAccount": true}}), + ); + add( + "x:Domain", + json!({"id": "d", "name": "example.org", "aliases": {"example.net": true}, + "isEnabled": true, "subAddressing": {"@type": "Enabled"}, + "catchAllAddress": "ann@example.org", "memberTenantId": "t"}), + ); + add( + "x:Domain", + json!({"id": "e", "name": "other.test", "isEnabled": false, + "description": "parked", "subAddressing": {"@type": "Custom", "customRule": "r"}}), + ); + add( + "x:Account", + json!({"id": "g", "@type": "Group", "name": "sales", "domainId": "d", + "roles": {"@type": "Default"}, "quotas": {}, "aliases": {}}), + ); + add( + "x:Account", + json!({"id": "a", "@type": "User", "name": "ann", "domainId": "d", + "description": "Ann", "memberGroupIds": {"g": true}, "memberTenantId": "t", + "roles": {"@type": "Custom", "roleIds": {"r2": true}}, + "permissions": {"@type": "Merge", "enabledPermissions": {"x": true}, "disabledPermissions": {}}, + "quotas": {"maxDiskQuota": 5000}, + "aliases": {"0": {"enabled": true, "name": "a", "domainId": "d", "description": "short"}, + "1": {"enabled": false, "name": "ann", "domainId": "e"}}, + "locale": "en_US", "timeZone": "America/Phoenix", "externalId": "emp-1"}), + ); + add( + "x:Account", + json!({"id": "b", "@type": "User", "name": "bob", "domainId": "e", + "roles": {"@type": "Admin"}, "quotas": {}, "aliases": {}}), + ); + add( + "x:MailingList", + json!({"id": "l", "name": "all", "domainId": "d", "description": "Everyone", + "recipients": {"ann@example.org": true, "outside@else.where": true}, + "aliases": {"0": {"enabled": true, "name": "everyone", "domainId": "d"}}}), + ); + s +} + +fn auth() -> Auth { + Auth::Basic { + user: "admin".into(), + password: "p".into(), + } +} + +fn export(registry: &Registry, dir: &Path) { + let summary = export_directory::run(ExportDirectoryConfig { + url: registry.url(), + auth: auth(), + dir: dir.to_path_buf(), + domains: Vec::new(), + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + }) + .expect("export"); + assert!(!summary.any_failed(), "{summary:?}"); +} + +fn import(registry: &Registry, dir: &Path, passwords: Option, dry_run: bool) -> Summary { + import_directory::run(ImportDirectoryConfig { + url: registry.url(), + auth: auth(), + dir: dir.to_path_buf(), + domains: Vec::new(), + passwords, + dry_run, + max_retries: 0, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + }) + .expect("import") +} + +fn counts<'a>(summary: &'a Summary, ty: &str) -> &'a TypeCounts { + &summary.per_type.iter().find(|(t, _)| *t == ty).unwrap().1 +} + +/// directory.json without what differs between any two runs. +fn directory(dir: &Path) -> Value { + let mut v: Value = + serde_json::from_str(&fs::read_to_string(dir.join("directory.json")).unwrap()).unwrap(); + for k in ["created", "createdBy", "source"] { + v.as_object_mut().unwrap().remove(k); + } + v +} + +#[test] +fn a_directory_imported_and_exported_again_comes_out_the_same() { + let tmp = TempDir::new().unwrap(); + let from = registry(source()); + let exported = tmp.path().join("from"); + export(&from, &exported); + + let to = registry(Store::default()); + let dry = import(&to, &exported, None, true); + assert_eq!(counts(&dry, "user").created, 2, "{dry:?}"); + assert_eq!(to.store.lock().unwrap().sets, 0, "a dry run sends no set"); + + let passwords = tmp.path().join("passwords.csv"); + let first = import(&to, &exported, Some(passwords.clone()), false); + assert!(!first.any_failed(), "{first:?}"); + for (ty, n) in [ + ("tenant", 1), + ("role", 2), + ("domain", 2), + ("group", 1), + ("user", 2), + ("mailinglist", 1), + ] { + assert_eq!(counts(&first, ty).created, n, "{ty}: {first:?}"); + } + + // Each user's stored password is the one written for it, privately. + let text = fs::read_to_string(&passwords).unwrap(); + let rows: BTreeMap<&str, &str> = text + .lines() + .skip(1) + .map(|l| l.split_once(',').unwrap()) + .collect(); + assert_eq!( + rows.keys().copied().collect::>(), + ["ann@example.org", "bob@other.test"] + ); + { + let s = to.store.lock().unwrap(); + for user in s.objects["x:Account"] + .iter() + .filter(|a| a["@type"] == "User") + { + let domain = s + .get("x:Domain", user["domainId"].as_str().unwrap()) + .unwrap(); + let email = format!( + "{}@{}", + user["name"].as_str().unwrap(), + domain["name"].as_str().unwrap() + ); + assert_eq!( + user["credentials"]["0"]["secret"], + rows[email.as_str()], + "{email}" + ); + } + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&passwords).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + } + + let again_dir = tmp.path().join("again"); + export(&to, &again_dir); + assert_eq!(directory(&again_dir), directory(&exported)); + + // Running again creates nothing and wants a new password file. + let err = import_directory::run(ImportDirectoryConfig { + url: to.url(), + auth: auth(), + dir: exported.clone(), + domains: Vec::new(), + passwords: Some(passwords.clone()), + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + }) + .expect_err("an existing password file is refused"); + assert!(err.to_string().contains("already exists"), "{err}"); + let second = import(&to, &exported, Some(tmp.path().join("second.csv")), false); + for (ty, _) in &second.per_type { + let c = counts(&second, ty); + assert_eq!((c.created, c.failed), (0, 0), "{ty}: {second:?}"); + } + assert_eq!( + fs::read_to_string(tmp.path().join("second.csv")).unwrap(), + "email,password\n" + ); +} + +#[test] +fn a_refused_account_is_counted_and_the_rest_are_made() { + let tmp = TempDir::new().unwrap(); + let mut src = source(); + src.objects.get_mut("x:Account").unwrap().push( + json!({"id": "z", "@type": "User", "name": "refused", "domainId": "d", + "roles": {"@type": "User"}, "quotas": {}, "aliases": {}}), + ); + let from = registry(src); + let exported = tmp.path().join("from"); + export(&from, &exported); + let to = registry(Store::default()); + let passwords = tmp.path().join("p.csv"); + let summary = import(&to, &exported, Some(passwords.clone()), false); + let users = counts(&summary, "user"); + assert_eq!((users.created, users.failed), (2, 1), "{summary:?}"); + let text = fs::read_to_string(&passwords).unwrap(); + assert!( + !text.contains("refused"), + "no password for an account not made: {text}" + ); +} -- 2.54.0