From 719a945d64db8856baafd1c689f295b453c39656 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sat, 22 Aug 2026 18:17:17 -0700 Subject: [PATCH] Initial commit: stalwart-migrator design and scaffolding In-place upgrade tool for Stalwart Mail Server (0.15.5 -> latest) with checkpointed rollback and post-migration validation. Design stage; see ARCHITECTURE.md. --- .gitignore | 26 + ARCHITECTURE.md | 530 ++++++++++++++++++++ README.md | 18 + go.mod | 3 + internal/backup/backup.go | 260 ++++++++++ internal/backup/backup_test.go | 217 ++++++++ internal/backup/binary.go | 31 ++ internal/backup/binary_test.go | 63 +++ internal/backup/doc.go | 3 + internal/backup/exectest_test.go | 48 ++ internal/backup/fdbbackup.go | 63 +++ internal/backup/fdbbackup_test.go | 48 ++ internal/backup/fsbackup.go | 205 ++++++++ internal/backup/fsbackup_test.go | 168 +++++++ internal/backup/hash.go | 25 + internal/backup/report.go | 36 ++ internal/backup/settingsdump.go | 164 ++++++ internal/backup/settingsdump_test.go | 175 +++++++ internal/backup/sqlbackup.go | 91 ++++ internal/backup/sqlbackup_test.go | 85 ++++ internal/backup/vandelay.go | 67 +++ internal/backup/vandelay_test.go | 56 +++ internal/checkpoint/doc.go | 3 + internal/checkpoint/step.go | 88 ++++ internal/checkpoint/store.go | 179 +++++++ internal/checkpoint/store_test.go | 168 +++++++ internal/checkpoint/types.go | 121 +++++ internal/config/doc.go | 3 + internal/plan/doc.go | 3 + internal/plan/plan.go | 122 +++++ internal/plan/plan_test.go | 71 +++ internal/preflight/checks.go | 290 +++++++++++ internal/preflight/checks_test.go | 304 +++++++++++ internal/preflight/cluster.go | 22 + internal/preflight/cluster_test.go | 34 ++ internal/preflight/deployment.go | 45 ++ internal/preflight/diskspace.go | 55 ++ internal/preflight/diskspace_test.go | 55 ++ internal/preflight/doc.go | 3 + internal/preflight/release.go | 74 +++ internal/preflight/release_test.go | 68 +++ internal/preflight/report.go | 46 ++ internal/preflight/storebackend.go | 106 ++++ internal/preflight/storebackend_test.go | 86 ++++ internal/preflight/version.go | 78 +++ internal/preflight/version_test.go | 58 +++ internal/recovery/apply.go | 54 ++ internal/recovery/apply_test.go | 60 +++ internal/recovery/doc.go | 3 + internal/recovery/exectest_test.go | 39 ++ internal/recovery/health.go | 44 ++ internal/recovery/main_test.go | 51 ++ internal/recovery/process.go | 96 ++++ internal/recovery/process_test.go | 104 ++++ internal/recovery/recovery.go | 107 ++++ internal/recovery/recovery_test.go | 113 +++++ internal/recovery/report.go | 31 ++ internal/rollback/doc.go | 3 + internal/stalwartapi/client.go | 73 +++ internal/stalwartapi/doc.go | 3 + internal/stalwartapi/mailbox.go | 112 +++++ internal/stalwartapi/mailbox_test.go | 112 +++++ internal/stalwartapi/management.go | 215 ++++++++ internal/stalwartapi/management_test.go | 201 ++++++++ internal/validate/bootcheck.go | 119 +++++ internal/validate/bootcheck_test.go | 251 +++++++++ internal/validate/content_integrity.go | 113 +++++ internal/validate/content_integrity_test.go | 165 ++++++ internal/validate/doc.go | 3 + internal/validate/main_test.go | 101 ++++ internal/validate/report.go | 40 ++ 71 files changed, 6677 insertions(+) create mode 100644 .gitignore create mode 100644 ARCHITECTURE.md create mode 100644 README.md create mode 100644 go.mod create mode 100644 internal/backup/backup.go create mode 100644 internal/backup/backup_test.go create mode 100644 internal/backup/binary.go create mode 100644 internal/backup/binary_test.go create mode 100644 internal/backup/doc.go create mode 100644 internal/backup/exectest_test.go create mode 100644 internal/backup/fdbbackup.go create mode 100644 internal/backup/fdbbackup_test.go create mode 100644 internal/backup/fsbackup.go create mode 100644 internal/backup/fsbackup_test.go create mode 100644 internal/backup/hash.go create mode 100644 internal/backup/report.go create mode 100644 internal/backup/settingsdump.go create mode 100644 internal/backup/settingsdump_test.go create mode 100644 internal/backup/sqlbackup.go create mode 100644 internal/backup/sqlbackup_test.go create mode 100644 internal/backup/vandelay.go create mode 100644 internal/backup/vandelay_test.go create mode 100644 internal/checkpoint/doc.go create mode 100644 internal/checkpoint/step.go create mode 100644 internal/checkpoint/store.go create mode 100644 internal/checkpoint/store_test.go create mode 100644 internal/checkpoint/types.go create mode 100644 internal/config/doc.go create mode 100644 internal/plan/doc.go create mode 100644 internal/plan/plan.go create mode 100644 internal/plan/plan_test.go create mode 100644 internal/preflight/checks.go create mode 100644 internal/preflight/checks_test.go create mode 100644 internal/preflight/cluster.go create mode 100644 internal/preflight/cluster_test.go create mode 100644 internal/preflight/deployment.go create mode 100644 internal/preflight/diskspace.go create mode 100644 internal/preflight/diskspace_test.go create mode 100644 internal/preflight/doc.go create mode 100644 internal/preflight/release.go create mode 100644 internal/preflight/release_test.go create mode 100644 internal/preflight/report.go create mode 100644 internal/preflight/storebackend.go create mode 100644 internal/preflight/storebackend_test.go create mode 100644 internal/preflight/version.go create mode 100644 internal/preflight/version_test.go create mode 100644 internal/recovery/apply.go create mode 100644 internal/recovery/apply_test.go create mode 100644 internal/recovery/doc.go create mode 100644 internal/recovery/exectest_test.go create mode 100644 internal/recovery/health.go create mode 100644 internal/recovery/main_test.go create mode 100644 internal/recovery/process.go create mode 100644 internal/recovery/process_test.go create mode 100644 internal/recovery/recovery.go create mode 100644 internal/recovery/recovery_test.go create mode 100644 internal/recovery/report.go create mode 100644 internal/rollback/doc.go create mode 100644 internal/stalwartapi/client.go create mode 100644 internal/stalwartapi/doc.go create mode 100644 internal/stalwartapi/mailbox.go create mode 100644 internal/stalwartapi/mailbox_test.go create mode 100644 internal/stalwartapi/management.go create mode 100644 internal/stalwartapi/management_test.go create mode 100644 internal/validate/bootcheck.go create mode 100644 internal/validate/bootcheck_test.go create mode 100644 internal/validate/content_integrity.go create mode 100644 internal/validate/content_integrity_test.go create mode 100644 internal/validate/doc.go create mode 100644 internal/validate/main_test.go create mode 100644 internal/validate/report.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b1dfadd --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Binaries +/bin/ +/dist/ +stalwart-migrate +*.exe + +# Test / coverage +*.out +*.test + +# Local state & logs +*.log +*.db +*.db-journal +*.db-wal +*.db-shm + +# Credentials / local config +.env +*.pem +*.key + +# Editor / OS +.DS_Store +.idea/ +.vscode/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..940012d --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,530 @@ +# stalwart-migrator — Architecture + +Status: design, no implementation yet. +Scope: upgrade a Stalwart Mail Server in place from **0.15.5** to the current +latest release (**0.16.14** as of 2026-08-19) with no data loss, a working +rollback at every step, and an automated post-migration validation pass. + +## 1. Why this isn't a thin wrapper + +Stalwart does not ship an automated upgrade tool today (planned for 1.0, +targeted H1 2026, not yet released). 0.15.5 → 0.16.x is a **major** boundary, +not a patch bump, and it is unusually dangerous to automate naively: + +- The v0.15 → v0.16 config model changes completely: multiple TOML files plus + DB-resident settings collapse into one `config.json` that describes only the + datastore connection, with everything else moved into JMAP-managed objects. +- Account names change from bare usernames to full email addresses; DAV URLs + change (`/dav/cal/alice` → `/dav/cal/alice%40example.com`). +- **On first v0.16 start, the server irreversibly deletes** all directory + records (users/groups/domains/tenants/OAuth clients), all settings, DMARC/ + TLS/ARF reports, pending tasks, telemetry, spam training samples, and quota + counters. Mail/calendar/contact data is untouched, but everything else is + gone unless captured first. +- Migration requires a manual "recovery mode" boot of the new binary, then an + external tool (`stalwart-cli apply`) replays a converted settings snapshot + into it over HTTP while it's up in that special mode — a multi-process, + multi-terminal, stateful procedure with no built-in resumability. +- In a cluster, every node must be stopped before migration starts; one node + left on v0.15 corrupts the shared store. +- Real-world failure mode already reported in the wild: post-migration WebUI + login breaks because the UI now requires HTTPS via `defaultHostname`, not + plain IP access — a config/DNS issue, not a data issue, but it reads as + "the migration broke everything" to an operator. +- Not all settings migrate automatically: SMTP listeners, routing, rate + limits, spam rules, and auth backends are explicitly **not** carried over + by Stalwart's own conversion script and must be recreated or replayed from + a separately captured snapshot. + +None of this is exotic — it's exactly what Stalwart's own +[`UPGRADING/v0_16.md`](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) +and `resources/scripts/migrate_v016.py` already do. This project's job is to +turn that fragile, manual, two-terminal runbook into a single supervised, +checkpointed, reversible operation — and to keep working as new releases land +on top of 0.16.x, most of which (0.16.1–0.16.14, per changelog) are pure +patch/feature releases with **no** schema migration, i.e. a binary swap + +smoke test, not a full migration. + +## 2. Design goals / non-goals + +**Goals** +- Zero data loss for mail, calendar, and contact content (the one thing + Stalwart itself guarantees is untouched — everything else is on us). +- Every phase has a defined, tested undo. Nothing destructive happens until + a verified backup exists. +- Fully automated happy path; the operator answers a preflight confirmation + once, then watches (or walks away and checks the report). +- Resumable: if the process dies mid-migration (crash, SSH drop, OOM), a + re-run picks up from the last completed checkpoint instead of redoing or, + worse, double-applying destructive steps. +- Works across the deployment shapes Stalwart actually supports: systemd + + bare binary, Docker/Compose, and single-node vs. cluster — with embedded + (RocksDB/SQLite) or external (PostgreSQL/MySQL/FoundationDB) stores. +- Extensible to future major boundaries (0.16 → 1.0 and beyond) without a + rewrite: version-boundary logic is pluggable, not hardcoded into the core + engine. + +**Non-goals** +- Not a general Stalwart config management tool (no drift detection, + no day-2 ops beyond the migration window). +- Not a replacement for routine backups — it *produces* a migration-time + backup as a side effect, but ongoing backup policy is the operator's job + (Stalwart's own guidance: import/export is explicitly not a backup + substitute; `Vandelay` per-account export is the documented backup tool). +- Not a cross-major-version skip tool. If the source is older than 0.15.x, + the tool requires stepping to 0.15.x first (this matches Stalwart's own + stated constraint — see UPGRADING notes). +- No support for editing mail content during migration (no format + conversion beyond what Stalwart's own store migration does). + +## 3. High-level flow + +``` + ┌─────────────┐ ┌───────────┐ ┌────────────┐ ┌───────────────┐ ┌────────────┐ ┌────────────┐ + │ PREFLIGHT │──▶│ BACKUP │──▶│ STAGE NEW │──▶│ RECOVERY-MODE │──▶│ CUTOVER │──▶│ VALIDATE │ + │ (checks, │ │ (defense │ │ BINARY + │ │ MIGRATE │ │ (swap, up, │ │ (functional│ + │ dry-run) │ │ in depth) │ │ config │ │ (apply plan) │ │ smoke) │ │ + counts) │ + └─────────────┘ └───────────┘ └────────────┘ └───────────────┘ └────────────┘ └────────────┘ + │ │ │ │ │ │ + └─────────────────┴────────────────┴─── on failure ──┴─────────────────┴──▶ ROLLBACK +``` + +Each box is a **phase**; each phase is a sequence of idempotent, checkpointed +**steps**. State is persisted to disk after every step (§5), so the whole +pipeline can be killed and re-invoked safely. + +For a pure patch bump within 0.16.x (no schema change per Stalwart's +changelog through 0.16.14), the plan collapses to: PREFLIGHT → BACKUP → +STAGE → CUTOVER → VALIDATE, skipping the recovery-mode phase entirely (see +§4.6). + +## 4. Phases + +### 4.1 Preflight + +Read-only. Aborts before touching anything if a hard blocker is found; +warns and asks for confirmation on soft blockers. + +- Detect current Stalwart version (`stalwart --version`, or JMAP + `Core/echo`/session endpoint if remote). +- Refuse to run if current version is outside the tool's supported starting + range (must be ≥0.15.0; older installs are told to upgrade to 0.15.x + first, per Stalwart's own guidance). +- Detect topology: systemd unit vs. Docker container vs. Compose, single + node vs. cluster member count (via config/cluster settings), reachable + peer nodes. +- **Cluster gate**: refuse to proceed unless every node in the cluster is + confirmed stopped (mirrors the documented hard requirement — one live + v0.15 node during migration corrupts the store). +- Detect store backend(s): RocksDB, SQLite, FoundationDB, PostgreSQL, + MySQL, plus configured blob store (local FS / S3-compatible) and FTS + backend (native / Elasticsearch). +- Disk space check: require free space ≥ N× current data directory size + (embedded stores need a full copy for the backup step; default threshold + configurable, hard-fail below a safety floor). +- Resolve and download the target binary/image, verify checksum/signature + against the published release. +- Fetch and pin the exact upstream `migrate_v016.py` (or its 0.16-successor + equivalent) revision, hash it, vendor the hash into the run's checkpoint + record — we depend on it as an external, versioned dependency, not a + static local copy that can silently drift from upstream. +- Dry-run the settings dump against the live server (read-only JMAP calls) + to confirm admin credentials and API reachability before anything + destructive is scheduled. +- Snapshot pre-migration *facts* used later for validation: account count, + per-account mailbox message counts (IMAP `STATUS`), domain list, DKIM key + fingerprints, TLS cert fingerprints, listener port list. Stored alongside + the checkpoint, not derived after the fact. +- Emit a plain-language plan summary and require explicit confirmation + (`--yes` to skip interactively, but never by default). + +### 4.2 Backup — defense in depth + +No single backup mechanism is trusted alone, because the risk profile is +different at each layer: + +1. **Filesystem/DB snapshot** (infra-level, fast, whole-store): + - Embedded (RocksDB/SQLite): stop-the-world `cp -a` of the data + directory (or LVM/ZFS snapshot if available — preferred, since it + doesn't require the copy to finish before the next step) to a sibling + path (`.v0155-backup`), never overwriting source. + - External SQL (Postgres/MySQL): targeted dump of the critical table + set Stalwart's own guide calls out (`s d r h b g j f u` for Postgres; + equivalent for MySQL), not a full-instance dump — matches the + documented, tested restore path and stays fast on large installs. + - FoundationDB: `fdbbackup` against the configured cluster. +2. **Settings/principals export** (the `migrate_v016.py dump` step): captured + during preflight *and* re-captured immediately before cutover, so the + export used for the apply reflects the last-known-good state, not a + stale preflight snapshot if time has passed. +3. **Per-account content export (Vandelay/JMAP)**: for installations under + an operator-configurable account-count threshold, take a belt-and-suspenders + full `vandelay import` (i.e. export-to-file) of every account into + self-contained per-account SQLite archives. This is independent of + storage backend and of the in-place migration path entirely — if + everything else somehow goes wrong, mail content is recoverable via + Stalwart's own documented import path into a clean instance. Skipped + above the threshold by default (time cost), but available as + `--full-content-backup` regardless of size. +4. **Binary preservation**: old binary is moved aside (`stalwart.v0155`), + never deleted, so rollback doesn't depend on re-downloading anything. + +Every backup artifact is checksummed and the checksum recorded in the +checkpoint file. Before moving past this phase, the tool **verifies** the +filesystem backup by opening it read-only with the *old* binary in a +throwaway temp directory and confirming it reports the expected version and +a sane account count — catching a corrupt or partial copy before it's relied +on, not after a failed rollback. + +### 4.3 Stage + +- Install target binary alongside the old one (never overwrite in place). +- Run `migrate_v016.py convert` against the fresh dump to produce + `config.json` + `export.json`, applying path rewrites for Docker/volume + layouts detected in preflight. +- Additionally generate an **apply-plan for the settings Stalwart's script + does not carry over** — SMTP listeners, routing rules, rate limits, spam + rules, auth backend config — by diffing the old effective config against + the new schema and emitting a best-effort JMAP object set for + `stalwart-cli apply`. This is flagged clearly as best-effort and included + in the final report for manual review; it's the one part of the + documented procedure that's explicitly manual today, and silently getting + it wrong (rather than flagging it) would be worse than not attempting it. +- Stage new systemd unit / Compose file changes without activating them. + +### 4.4 Recovery-mode migration + +This is the phase most exposed to partial-failure — it drives an external +process (the new Stalwart binary) through an undocumented-duration startup, +then drives a second external process (`stalwart-cli apply`) against it over +HTTP. Both are supervised with explicit timeouts and health polling, not +fire-and-forget: + +1. Stop the old service. +2. Start the new binary in the foreground with + `STALWART_RECOVERY_MODE=1` and a freshly generated one-time + `STALWART_RECOVERY_ADMIN` credential (random, never the operator's real + password, never logged). +3. Poll the recovery HTTP endpoint until healthy or a timeout elapses; on + timeout, capture logs and fail into the rollback path rather than + hanging indefinitely. +4. Run `stalwart-cli apply --file export.json`, then the generated + best-effort settings plan from §4.3, capturing full output. +5. Verify the apply reported success for every object (the tool parses the + apply-tool's structured output rather than trusting exit code alone — + partial application with a zero exit code is exactly the kind of silent + failure this tool exists to catch). +6. Stop recovery mode cleanly (SIGTERM, not SIGKILL, to let it flush). + +Checkpointed after each numbered step, so a crash between "apply succeeded" +and "recovery mode stopped" resumes at step 6 instead of re-running apply +against an already-migrated store. + +### 4.5 Cutover + +- Update the real systemd unit / Compose config to point at the new binary + and config, without the recovery env vars (leaving + `STALWART_RECOVERY_MODE=1` set is a documented footgun — it would recovery- + boot on every restart). +- Start the service normally. +- Wait for healthy JMAP session response. +- Trigger disk-quota (and tenant-quota, if multi-tenant) recalculation via + the management API, and poll the task queue until it completes rather + than firing and moving on. + +### 4.6 Patch-bump fast path + +For an already-0.16.x install moving to a newer 0.16.x patch (the common +case after the initial major migration, and per the changelog the case for +every release from 0.16.1 through 0.16.14 so far): preflight confirms no +schema-migration flag is set for the target version, and the plan skips +§4.4 entirely — binary swap, restart, same validation suite as §4.7. This +is intentionally the same engine with a shorter plan, not a separate +code path, so it doesn't rot independently. + +### 4.7 Post-migration validation + +Runs automatically after cutover; failure here triggers rollback (§4.8) +unless `--no-auto-rollback` was passed, in which case it just reports and +exits non-zero. + +- **Version check**: reported server version matches the target exactly. +- **Auth check**: WebUI login succeeds over the *configured hostname* via + HTTPS (not bare IP) — this directly targets the real-world post-0.16 + login failure mode found in the field. +- **Protocol reachability**: JMAP session, IMAP, SMTP (submission + MTA), + POP3, ManageSieve, CalDAV/CardDAV endpoints all accept a handshake on + their configured ports. +- **Directory integrity**: account/domain/group counts match the preflight + snapshot exactly (accounting for the bare-username → email-address + rewrite, which the tool resolves by comparing normalized identities, not + raw strings). +- **Content integrity — the core no-data-loss check**: per-account IMAP + `STATUS (MESSAGES)` compared against the preflight snapshot for every + mailbox of every account (or a statistically sampled subset above a + configurable account-count threshold, with the full sweep always + available via `--full-validation`). Any mismatch is a hard failure. +- **DKIM/TLS check**: key fingerprints and cert validity match or are + intentionally rotated (new-key generation is an expected v0.16 behavior, + not a bug — the check distinguishes "changed as documented" from + "missing"). +- **DNS check**: for domains under Stalwart's automatic DNS management + (new in 0.16), diff expected vs. actual published records and flag + drift rather than assume the automation ran correctly. +- **Mail-flow smoke test**: send one real message through SMTP submission + to a dedicated canary mailbox and confirm it's retrievable via IMAP + within a timeout — the one end-to-end check that nothing upstream can + fake. +- **Quota check**: recalculation task (§4.5) completed and reported + numbers are non-zero/sane where preflight showed non-zero usage. + +Output is a single structured report (JSON + human summary): pass/fail per +check, with enough detail to hand to the operator or to a rollback decision. + +### 4.8 Rollback + +Two triggers: automatic (validation failure, unless disabled) or manual +(`stalwart-migrate rollback `, usable any time up to a +"rollback window closed" checkpoint the operator explicitly confirms once +they're satisfied — see §6). + +Procedure, checkpoint-resumable like everything else: +1. Stop the new service (or recovery-mode process, if failure happened + there). +2. Restore the filesystem/DB backup from §4.2 to the original path + (`.v0155-backup` → ``), or restore the targeted SQL + dump for external databases. +3. Restore the old systemd unit / Compose config. +4. Restart the preserved old binary. +5. Re-run a reduced version of the §4.7 validation suite against the + *restored* instance (version check, protocol reachability, directory + counts) to confirm rollback actually worked rather than assuming it did. +6. Report clearly that the instance is back on 0.15.5 and the new-version + artifacts (staged binary, export.json, apply-plan) are preserved + untouched for a retry after the underlying issue is fixed. + +Rollback never deletes anything from the failed attempt — a second forward +attempt reuses the existing backup and dumps rather than re-capturing +(faster retry, and one fewer chance for the retry's own backup step to +fail). + +**Status: not yet implemented.** `stalwart-migrate run` without `--dry-run` +currently refuses to proceed, precisely because this phase doesn't exist yet +— committing to a real cutover without a working rollback would violate the +one guarantee this tool exists to provide. §4.9 covers what does work today. + +### 4.9 Dry run + +`stalwart-migrate run --dry-run` runs the real migration mechanics against a +disposable sandbox clone of the data, so an operator can get genuine +confidence *before* committing to a real cutover — not a simulation that +skips the fragile parts, the actual recovery-mode migration (§4.4) and a +post-migration boot check, just pointed somewhere disposable: + +1. **Preflight** (§4.1) runs for real, read-only, against the live instance. +2. **Backup** (§4.2) runs for real too, with one exception: + `SkipBinaryPreservation` is set, so the production binary at the real + install path is never moved aside. Taking a *consistent* filesystem + snapshot of an embedded store still means the live service should be + stopped first (the same requirement Stalwart's own export tooling has) — + this tool doesn't automate that stop/start today (no systemd/Docker + control exists yet), so a dry-run without a manual stop first is a + best-effort snapshot of a live, in-use store, and the CLI says so. +3. **Convert**: `migrate_v016.py convert` turns the settings/principals dump + into `config.json` + `export.json`, using the script's own documented + `--patch-paths =` flag to point the generated config at the + sandbox data directory instead of the real one. This is the officially + documented mechanism for exactly this kind of path redirection — the tool + deliberately does not try to rewrite `config.json`'s contents itself, + since depending on its exact schema (which has already changed once, + 0.15 → 0.16) is a correctness risk this tool avoids wherever an official + alternative exists. +4. The verified backup copy is cloned again into the sandbox directory + (never reusing the same directory recovery mode is about to mutate as the + one rollback would restore from). +5. **Recovery-mode migration** (§4.4) runs for real against the sandbox: + the actual target binary, actual `STALWART_RECOVERY_MODE=1` boot, actual + `stalwart-cli apply`. +6. **Boot check + content integrity**: the migrated sandbox is started once + more as an ordinary boot (no recovery-mode env vars) and polled until its + HTTP listener answers, confirming the migrated store doesn't just accept + a settings apply but actually comes up cleanly afterward. If preflight + captured a pre-migration snapshot (§4.1, requires `--admin-url`), the + same boot is then used to capture a fresh post-migration snapshot and + compare the two — this is the actual no-data-loss guarantee, not just + "the mechanics ran": every account and mailbox from before must still be + found afterward (matching by exact name, falling back to the part before + `@` since v0.16's own migration rewrites bare usernames to full email + addresses) with an identical message count. A mismatch or a missing + account fails the check. This covers the message-count half of §4.7's + full suite; DKIM/TLS fingerprint checks and a live mail-flow SMTP→IMAP + smoke test are still open. +7. Every byte written by steps 2–6 (the fs-backup copy, settings/principals + dumps, downloaded `migrate_v016.py`, sandbox clone, and generated + `config.json`/`export.json`) lives under one per-run directory + (`work-dir/`), which is removed on *every* exit path - success, + a failed check partway through, or an early refusal - via a deferred + cleanup, not just the happy path. The only thing left behind afterward + is the checkpoint's `state.json` under `--state-dir`: a small structured + success/failure log (which check failed and why), not bulk data. + `--keep-artifacts` opts out for inspecting a failure. Nothing at the real + binary path, the real service, or the real data directory's *contents* + is ever mutated by steps 2–6 in the first place. + +A same-boundary patch bump (§4.6) has no recovery phase to simulate — dry +run for that plan is just preflight + backup. + +## 5. State machine / checkpointing + +Every run gets a `run-id` and a checkpoint file +(`/var/lib/stalwart-migrator/runs//state.json`) written after each +step completes, containing: run-id, source/target version, current +phase/step, timestamps, artifact paths + checksums, and the preflight +snapshot facts used by validation. Steps are pure functions of +(checkpoint-state → new-state); re-invoking `stalwart-migrate run` with an +in-progress run-id resumes at the first incomplete step. Steps are written +to be safe to re-run if they were interrupted mid-execution (e.g. the +filesystem copy step checks for and resumes/redoes a partial copy rather +than trusting a checkpoint that says "started" as if it meant "done"). + +This is the same shape as a deployment pipeline's state file, deliberately — +the risk profile (long-running, multi-process, must survive being killed) +is the same problem. + +## 6. CLI surface + +``` +stalwart-migrate preflight [--config PATH] ... # read-only, prints the report +stalwart-migrate run --dry-run [--target-binary PATH] ... # implemented — see §4.9 + [--keep-artifacts] + (without --dry-run: refused today — see §4.8's status note) +stalwart-migrate status [run-id] # implemented +stalwart-migrate rollback # not yet implemented (§4.8) +stalwart-migrate confirm # not yet implemented +stalwart-migrate report [--json] # not yet implemented +``` + +`run` is the only command that mutates anything and it always starts with +preflight. Once rollback exists, `confirm` will be a separate, explicit step +so backups aren't pruned just because validation passed automatically — the +operator gets a beat to actually use the migrated server before disk space +is reclaimed. Default retention if never confirmed: configurable TTL, warns +loudly, never auto-deletes silently. Flags shown here are the design intent; +run `stalwart-migrate -h` for the actual current flag set. + +## 7. Project layout (Go, matches this workspace's other CLI tools) + +``` +stalwart-migrator/ + cmd/stalwart-migrate/ main.go, preflight.go, run.go, status.go — CLI entry + wiring + internal/plan/ version-boundary → ordered phase list (§4.6) [done] + internal/checkpoint/ run-id, state.json read/write, resume logic (§5) [done] + internal/preflight/ §4.1 checks [done] + internal/backup/ §4.2 — fs/db snapshot, settings dump+convert, Vandelay export [done] + internal/recovery/ §4.4 — recovery-mode process supervision + apply [done] + internal/validate/ §4.7 — boot-check + content-integrity done; DKIM/TLS + mail-flow not yet [partial] + internal/rollback/ §4.8 [not started] + internal/stalwartapi/ Ping + AccountSnapshot, incl. per-mailbox counts via impersonation (§8) [done] + internal/config/ tool's own config (paths, thresholds, credentials handling) [not started] + docs/ this file + phase-specific notes as they get built out +``` + +There's no separate `internal/stage` package: the `convert` half of +`migrate_v016.py` lives in `internal/backup` next to `dump` (same script, +same invocation pattern), and the dry-run sandbox-cloning logic that stands +in for the rest of §4.3 currently lives directly in `cmd/run.go` rather than +its own package, pending a real cutover phase to generalize it against. + +`internal/stalwartapi` is deliberately the only thing that speaks JMAP/HTTP +to Stalwart — every other package depends on it, not on `net/http` directly, +so auth handling and retry/backoff live in one place. + +## 8. Open questions for the next pass + +- **Credential handling**: recovery-mode admin password and any stored JMAP + credentials need a real secrets story (env var pass-through is fine for + v1, but the checkpoint file must never contain them in plaintext). +- **Cluster orchestration**: §4.1's cluster gate assumes the operator stops + other nodes manually; a v2 could SSH-coordinate that instead. Out of scope + for v1. +- **`migrate_v016.py` dependency**: pinning by hash is a start, but the + script is Stalwart's, not ours — need a policy for what happens when it + changes upstream (re-vendor + re-test before bumping the pin, never + silently float to `main`). +- **Best-effort settings apply-plan (§4.3)**: needs real-world testing + against a variety of existing SMTP/routing/spam configs before it's + trusted un-reviewed; v1 should probably always require operator sign-off + on that specific generated plan even with `--yes` set for everything else. + Not started — dry-run currently only replays what `migrate_v016.py` + itself converts. +- **Account/mailbox enumeration** (`stalwartapi.Client.AccountSnapshot`): + **implemented**, including per-mailbox message counts. Account count and + domains come from `x:Account/query` + `x:Account/get` against Stalwart's + management API (`/api`, capability `urn:stalwart:jmap`), confirmed + against `crates/jmap/src/principal/{get,query}.rs` and + `docs/ref/object/account.md`. + Per-mailbox counts needed a second research pass, because a superuser's + own JMAP session does **not** implicitly grant cross-account access — + confirmed by reading `crates/jmap/src/api/session.rs`: the session's + `accounts` map is built solely from the authenticated identity's own + membership/sharing grants, unaffected by any admin flag. The real, + documented mechanism is Stalwart's `impersonate` permission + (`docs/auth/authorization/administrator.md`): an account holding it can + log in *as* another account via the composite Basic-auth username + `%`, after which standard RFC 8621 `Mailbox/get` + (property `totalEmails`) works normally against that impersonated + session's own `apiUrl` (session-discovered per RFC 8620, not the `/api` + management endpoint — confirmed as a distinct endpoint in + `docs/ref/object/account.md`). `AccountSnapshot` now does this per + account it finds; a single account's failure (most likely: `impersonate` + not granted) is recorded in `Snapshot.MailboxErrors` rather than failing + the whole snapshot, so one misconfigured account doesn't hide a working + result for every other one. + One resolved false alarm worth recording: an initial pass of this same + research, reading Stalwart's `main` branch source directly, reported + `x:Account` apparently replaced by `x:Principal`/`x:Quota`. Checking the + *published* docs site directly (which has no `principal.md`/`quota.md` + page, and still documents `x:Account` with a working example) showed that + was an unreleased/in-development refactor in `main`, not the interface + the current released version actually exposes — a reminder that "read the + source" and "read what's actually shipped" can disagree, and it's worth + checking both before changing already-working code on the strength of one. + Preflight now populates `RunState.PreflightSnapshot.MailboxCounts` when + `--admin-url` is set, and `validate.BootCheck` now compares it against a + fresh post-migration snapshot as part of the same boot (§4.9 step 6) — + proven end-to-end with a live smoke test that deliberately made the + "after" instance report fewer messages than the "before" snapshot and + confirmed the dry run failed loudly with the exact before/after counts, + rather than just trusting that. **Still open**: preflight/validate always + attempt every account serially with no sampling/threshold, which could be + slow on a large install — `--full-validation`'s sampling idea from §4.7 + hasn't been built yet for this; and DKIM/TLS fingerprint checks plus a + live mail-flow SMTP→IMAP smoke test (the rest of §4.7's suite) aren't + implemented. With this done, recovery, backup, dry-run, and account/ + mailbox snapshotting all work end-to-end, and dry-run's comparison is now + the closest thing to §4.7's actual no-data-loss guarantee this tool has — + the remaining major gap is `internal/rollback` and real cutover (below). +- **No real cutover or rollback yet**: `internal/rollback` doesn't exist, + and neither does any systemd/Docker service control. `run` without + `--dry-run` refuses for exactly this reason. Building rollback first - + before wiring a real cutover - is deliberate: this tool should never be + able to commit to a change it can't undo. +- **Dry-run's un-stopped backup snapshot** (§4.9 step 2): without service + control, a dry-run backs up a live, in-use store unless the operator stops + it manually first. Worth revisiting once service control exists, so + dry-run can offer to do this safely itself. + +## Sources + +Grounded in Stalwart's own documentation and community reports as of +2026-08-19: +- [UPGRADING/v0_16.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) — exact migration procedure this tool automates +- [UPGRADING/v0_15.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) — prior breaking-change boundary +- [Database Migration docs](https://stalw.art/docs/management/maintenance/migration/) +- [Backup docs](https://stalw.art/docs/migration/import-export/backup/) (Vandelay) +- [Upgrading guide](https://stalw.art/docs/install/upgrade/) +- [v0.16 blog post](https://stalw.art/blog/stalwart-0-16/) +- [Discussion #2892](https://github.com/stalwartlabs/stalwart/discussions/2892) — breaking changes overview +- [Discussion #3004](https://github.com/stalwartlabs/stalwart/discussions/3004) — upgrading Q&A +- [Discussion #3025](https://github.com/stalwartlabs/stalwart/discussions/3025) — real post-migration WebUI login failure +- [CHANGELOG.md](https://github.com/stalwartlabs/stalwart/blob/main/CHANGELOG.md) — confirms 0.16.1–0.16.14 carry no further schema migrations diff --git a/README.md b/README.md new file mode 100644 index 0000000..60c9b1c --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# stalwart-migrator + +In-place upgrade tool for Stalwart Mail Server: 0.15.5 → latest, with no +data loss, checkpointed rollback at every step, and automated post-migration +validation. + +Design status: architected, not yet implemented — see +[ARCHITECTURE.md](ARCHITECTURE.md) for the full design, the phase-by-phase +plan, and the research it's grounded in. + +``` +stalwart-migrate preflight # read-only checks + plan +stalwart-migrate run # execute the migration +stalwart-migrate status +stalwart-migrate rollback +stalwart-migrate confirm +stalwart-migrate report +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..869cfdd --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/johnellis/stalwart-migrator + +go 1.26.5 diff --git a/internal/backup/backup.go b/internal/backup/backup.go new file mode 100644 index 0000000..5cc57bb --- /dev/null +++ b/internal/backup/backup.go @@ -0,0 +1,260 @@ +package backup + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" +) + +// Options configures a full backup pass. Which fields matter depends on +// rs.Topology.StoreBackend, as recorded by the preflight phase - Run +// branches on that rather than requiring the caller to pre-select a code +// path. +type Options struct { + // Binary preservation. + BinaryPath string + // SkipBinaryPreservation, when true, never touches BinaryPath - used by + // a dry run, which must not move the production binary aside just to + // simulate a migration it hasn't committed to. + SkipBinaryPreservation bool + + // Embedded backend (RocksDB/SQLite). + DataDir string + BackupDir string // destination for the fs snapshot; required if the backend is embedded + + // External SQL backend (PostgreSQL/MySQL). + SQL SQLOptions + + // FoundationDB backend. + FDB FDBOptions + + // Settings/principals dump (always runs - every topology needs it). + MigrationScriptURL string // defaults to DefaultMigrationScriptURL + MigrationScriptSHA256 string // pinned hash; empty accepts and reports whatever is fetched (see DownloadFile) + ScriptDestPath string + AdminURL string + AdminUser string + AdminPassword string + SettingsDumpPath string + PrincipalsDumpPath string + PythonPath string + HTTPClient *http.Client + + // Per-account content export (Vandelay) - optional defense-in-depth + // layer. Empty Accounts skips it entirely; this is expected until + // account enumeration is wired up (see stalwartapi.Client.AccountSnapshot). + Vandelay VandelayOptions + Accounts []string +} + +// Run executes the full backup pass described in ARCHITECTURE.md §4.2, +// checkpointing each step. Unlike preflight, most steps here are hard +// failures: a backup that didn't actually happen must stop the pipeline, +// not just get reported and continued past. The one genuinely optional +// layer is the per-account Vandelay export, which is skipped (Status: +// StatusSkipped, not a failure) when Options.Accounts is empty - but if the +// operator populated Accounts, a failure there is hard too, since silently +// downgrading an explicitly requested backup layer would be worse than +// stopping. +func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (Report, error) { + var report Report + + step := func(name string, fn func() (checkpoint.StepOutcome, error)) (checkpoint.StepOutcome, error) { + outcome, err := store.RunStep(rs, checkpoint.PhaseBackup, name, fn) + if err != nil { + report.Results = append(report.Results, CheckResult{Name: name, Status: StatusFail, Detail: err.Error()}) + return outcome, err + } + status := Status(outcome.Verdict) + if status == "" { + status = StatusOK + } + report.Results = append(report.Results, CheckResult{Name: name, Status: status, Detail: outcome.Detail}) + return outcome, nil + } + + if opts.SkipBinaryPreservation { + report.Results = append(report.Results, CheckResult{ + Name: "preserve-binary", Status: StatusSkipped, + Detail: "skipped (SkipBinaryPreservation) - the production binary is never touched, e.g. for a dry run", + }) + } else if _, err := step("preserve-binary", func() (checkpoint.StepOutcome, error) { + preserved, err := PreserveBinary(opts.BinaryPath, rs.SourceVersion) + if err != nil { + return checkpoint.StepOutcome{}, err + } + sum, size, err := hashFile(preserved) + if err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("backup: hash preserved binary %s: %w", preserved, err) + } + rs.RecordArtifact("old-binary", checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size}) + return checkpoint.StepOutcome{Detail: fmt.Sprintf("preserved %s as %s", opts.BinaryPath, preserved), Extra: preserved}, nil + }); err != nil { + return report, err + } + + backends := strings.ToLower(rs.Topology.StoreBackend) + + switch { + case strings.Contains(backends, "rocksdb") || strings.Contains(backends, "sqlite"): + if _, err := step("fs-snapshot", func() (checkpoint.StepOutcome, error) { + manifest, err := CopyDataDir(opts.DataDir, opts.BackupDir) + if err != nil { + return checkpoint.StepOutcome{}, err + } + manifestPath := filepath.Join(opts.BackupDir, "..", filepath.Base(opts.BackupDir)+".manifest.json") + if err := WriteManifest(manifestPath, manifest); err != nil { + return checkpoint.StepOutcome{}, err + } + sum, err := manifest.Checksum() + if err != nil { + return checkpoint.StepOutcome{}, err + } + rs.RecordArtifact("fs-backup", checkpoint.Artifact{Path: opts.BackupDir, SHA256: sum, SizeBytes: manifest.TotalBytes}) + return checkpoint.StepOutcome{ + Detail: fmt.Sprintf("copied %d file(s), %d bytes, from %s to %s", len(manifest.Files), manifest.TotalBytes, opts.DataDir, opts.BackupDir), + Extra: manifestPath, + }, nil + }); err != nil { + return report, err + } + + if _, err := step("fs-verify", func() (checkpoint.StepOutcome, error) { + manifestOutcome := rs.Outcome(checkpoint.PhaseBackup, "fs-snapshot") + manifest, err := ReadManifest(manifestOutcome.Extra) + if err != nil { + return checkpoint.StepOutcome{}, err + } + if err := VerifyDataDirBackup(opts.BackupDir, manifest); err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("re-hashed %d file(s), all match the manifest recorded at copy time", len(manifest.Files))}, nil + }); err != nil { + return report, err + } + + case strings.Contains(backends, "postgresql"): + if _, err := step("sql-dump", func() (checkpoint.StepOutcome, error) { + if err := RunPgDump(ctx, opts.SQL); err != nil { + return checkpoint.StepOutcome{}, err + } + sum, size, err := hashFile(opts.SQL.OutPath) + if err != nil { + return checkpoint.StepOutcome{}, err + } + rs.RecordArtifact("sql-dump", checkpoint.Artifact{Path: opts.SQL.OutPath, SHA256: sum, SizeBytes: size}) + return checkpoint.StepOutcome{Detail: fmt.Sprintf("pg_dump of critical tables (%s) to %s, %d bytes", strings.Join(criticalTables, " "), opts.SQL.OutPath, size)}, nil + }); err != nil { + return report, err + } + + case strings.Contains(backends, "mysql"): + if _, err := step("sql-dump", func() (checkpoint.StepOutcome, error) { + if err := RunMySQLDump(ctx, opts.SQL); err != nil { + return checkpoint.StepOutcome{}, err + } + sum, size, err := hashFile(opts.SQL.OutPath) + if err != nil { + return checkpoint.StepOutcome{}, err + } + rs.RecordArtifact("sql-dump", checkpoint.Artifact{Path: opts.SQL.OutPath, SHA256: sum, SizeBytes: size}) + return checkpoint.StepOutcome{Detail: fmt.Sprintf("mysqldump of critical tables (%s) to %s, %d bytes", strings.Join(criticalTables, " "), opts.SQL.OutPath, size)}, nil + }); err != nil { + return report, err + } + + case strings.Contains(backends, "foundationdb"): + if _, err := step("fdb-backup", func() (checkpoint.StepOutcome, error) { + if err := StartFDBBackup(ctx, opts.FDB); err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{ + Detail: fmt.Sprintf("fdbbackup start issued for destination %s (tag %s) - this only confirms the job was accepted, not that it finished; check `fdbbackup status` before relying on it", opts.FDB.Destination, opts.FDB.Tag), + }, nil + }); err != nil { + return report, err + } + + default: + report.Results = append(report.Results, CheckResult{ + Name: "backend-backup", Status: StatusSkipped, + Detail: fmt.Sprintf("no known store backend recorded for this run (topology.store_backend=%q) - preflight must run first, or the backend wasn't recognized; no filesystem/DB backup was taken", rs.Topology.StoreBackend), + }) + } + + if _, err := step("settings-dump", func() (checkpoint.StepOutcome, error) { + scriptURL := opts.MigrationScriptURL + if scriptURL == "" { + scriptURL = DefaultMigrationScriptURL + } + sum, err := DownloadFile(ctx, opts.HTTPClient, scriptURL, opts.ScriptDestPath, opts.MigrationScriptSHA256) + if err != nil { + return checkpoint.StepOutcome{}, err + } + pinNote := "" + if opts.MigrationScriptSHA256 == "" { + pinNote = fmt.Sprintf(" (no pin was configured - record sha256 %s as MigrationScriptSHA256 to pin it for future runs)", sum) + } + if err := RunSettingsDump(ctx, SettingsDumpOptions{ + PythonPath: opts.PythonPath, + ScriptPath: opts.ScriptDestPath, + URL: opts.AdminURL, + Username: opts.AdminUser, + Password: opts.AdminPassword, + SettingsPath: opts.SettingsDumpPath, + PrincipalsPath: opts.PrincipalsDumpPath, + }); err != nil { + return checkpoint.StepOutcome{}, err + } + settingsSum, settingsSize, err := hashFile(opts.SettingsDumpPath) + if err != nil { + return checkpoint.StepOutcome{}, err + } + principalsSum, principalsSize, err := hashFile(opts.PrincipalsDumpPath) + if err != nil { + return checkpoint.StepOutcome{}, err + } + rs.RecordArtifact("settings-dump", checkpoint.Artifact{Path: opts.SettingsDumpPath, SHA256: settingsSum, SizeBytes: settingsSize}) + rs.RecordArtifact("principals-dump", checkpoint.Artifact{Path: opts.PrincipalsDumpPath, SHA256: principalsSum, SizeBytes: principalsSize}) + return checkpoint.StepOutcome{ + Detail: fmt.Sprintf("dumped settings (%d bytes) and principals (%d bytes) from %s%s", settingsSize, principalsSize, opts.AdminURL, pinNote), + Extra: sum, + }, nil + }); err != nil { + return report, err + } + + if len(opts.Accounts) == 0 { + report.Results = append(report.Results, CheckResult{ + Name: "vandelay-export", Status: StatusSkipped, + Detail: "no account list supplied - skipped; pass Options.Accounts (or --full-content-backup with an account source once account enumeration is wired up) to enable this belt-and-suspenders layer", + }) + } else if _, err := step("vandelay-export", func() (checkpoint.StepOutcome, error) { + if err := os.MkdirAll(opts.Vandelay.OutDir, 0o750); err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("backup: create vandelay output dir %s: %w", opts.Vandelay.OutDir, err) + } + files, err := ExportAccounts(ctx, opts.Vandelay, opts.Accounts) + if err != nil { + return checkpoint.StepOutcome{}, err + } + var totalSize int64 + for i, f := range files { + sum, size, err := hashFile(f) + if err != nil { + return checkpoint.StepOutcome{}, err + } + rs.RecordArtifact(fmt.Sprintf("vandelay-%s", opts.Accounts[i]), checkpoint.Artifact{Path: f, SHA256: sum, SizeBytes: size}) + totalSize += size + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("exported %d account(s), %d bytes total, to %s", len(files), totalSize, opts.Vandelay.OutDir)}, nil + }); err != nil { + return report, err + } + + return report, nil +} diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go new file mode 100644 index 0000000..0da492b --- /dev/null +++ b/internal/backup/backup_test.go @@ -0,0 +1,217 @@ +package backup + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" +) + +func TestBackupRunEndToEndAndResume(t *testing.T) { + dir := t.TempDir() + + binaryPath := filepath.Join(dir, "stalwart") + if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil { + t.Fatal(err) + } + + dataDir := filepath.Join(dir, "data") + if err := os.MkdirAll(dataDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dataDir, "db.bin"), []byte("some rocksdb-shaped data"), 0o644); err != nil { + t.Fatal(err) + } + backupDir := filepath.Join(dir, "data-backup") + + settingsPath := filepath.Join(dir, "settings.json") + principalsPath := filepath.Join(dir, "principals.json") + pythonScript := fmt.Sprintf("#!/bin/sh\necho fake-settings > %q\necho fake-principals > %q\n", settingsPath, principalsPath) + pythonDir := withFakeExecutable(t, "python3", pythonScript) + pythonScriptPath := filepath.Join(pythonDir, "python3") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("print('fake migrate_v016.py')\n")) + })) + defer srv.Close() + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("store.Create: %v", err) + } + rs.Topology = checkpoint.Topology{StoreBackend: "rocksdb"} + + opts := Options{ + BinaryPath: binaryPath, + DataDir: dataDir, + BackupDir: backupDir, + MigrationScriptURL: srv.URL, + ScriptDestPath: filepath.Join(dir, "migrate_v016.py"), + AdminURL: "https://mail.example.com", + AdminUser: "admin", + AdminPassword: "hunter2", + SettingsDumpPath: settingsPath, + PrincipalsDumpPath: principalsPath, + } + + report, err := Run(context.Background(), store, rs, opts) + if err != nil { + t.Fatalf("Run #1: %v", err) + } + for _, r := range report.Results { + if r.Status == StatusFail { + t.Errorf("Run #1: step %s failed: %s", r.Name, r.Detail) + } + } + for _, name := range []string{"old-binary", "fs-backup", "settings-dump", "principals-dump"} { + if _, ok := rs.Artifacts[name]; !ok { + t.Errorf("Run #1: expected artifact %q to be recorded, got %v", name, rs.Artifacts) + } + } + if _, err := os.Stat(binaryPath + ".v0.15.5"); err != nil { + t.Errorf("preserved binary missing: %v", err) + } + + // -- simulate a crash and resume, making every non-idempotent step's + // -- inputs unusable so a re-execution (rather than a skip) fails loudly. + if err := os.RemoveAll(dataDir); err != nil { + t.Fatal(err) + } + srv.Close() + if err := os.WriteFile(pythonScriptPath, []byte("#!/bin/sh\necho should-not-run-again >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + + resumed, err := store.Load(rs.RunID) + if err != nil { + t.Fatalf("store.Load (resume): %v", err) + } + report2, err := Run(context.Background(), store, resumed, opts) + if err != nil { + t.Fatalf("Run #2 (resume) should succeed without redoing completed steps: %v", err) + } + if len(report2.Results) != len(report.Results) { + t.Errorf("resumed report has %d results, want %d (same as first run)", len(report2.Results), len(report.Results)) + } + for _, r := range report2.Results { + if r.Status == StatusFail { + t.Errorf("Run #2 (resume): step %s unexpectedly failed: %s", r.Name, r.Detail) + } + } +} + +func TestBackupRunSkipsVandelayWhenNoAccountsGiven(t *testing.T) { + dir := t.TempDir() + binaryPath := filepath.Join(dir, "stalwart") + os.WriteFile(binaryPath, []byte("fake binary"), 0o755) + dataDir := filepath.Join(dir, "data") + os.MkdirAll(dataDir, 0o755) + os.WriteFile(filepath.Join(dataDir, "db.bin"), []byte("x"), 0o644) + + withFakeExecutable(t, "python3", fmt.Sprintf( + "#!/bin/sh\necho x > %q\necho x > %q\n", + filepath.Join(dir, "settings.json"), filepath.Join(dir, "principals.json"), + )) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("script")) + })) + defer srv.Close() + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatal(err) + } + rs.Topology = checkpoint.Topology{StoreBackend: "rocksdb"} + + opts := Options{ + BinaryPath: binaryPath, + DataDir: dataDir, + BackupDir: filepath.Join(dir, "data-backup"), + MigrationScriptURL: srv.URL, + ScriptDestPath: filepath.Join(dir, "migrate_v016.py"), + AdminURL: "https://mail.example.com", + SettingsDumpPath: filepath.Join(dir, "settings.json"), + PrincipalsDumpPath: filepath.Join(dir, "principals.json"), + // Accounts intentionally left empty. + } + + report, err := Run(context.Background(), store, rs, opts) + if err != nil { + t.Fatalf("Run: %v", err) + } + found := false + for _, r := range report.Results { + if r.Name == "vandelay-export" { + found = true + if r.Status != StatusSkipped { + t.Errorf("vandelay-export status = %s, want skipped", r.Status) + } + } + } + if !found { + t.Error("report missing a vandelay-export entry") + } +} + +func TestBackupRunSkipsBinaryPreservationForDryRun(t *testing.T) { + dir := t.TempDir() + binaryPath := filepath.Join(dir, "stalwart") + if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil { + t.Fatal(err) + } + dataDir := filepath.Join(dir, "data") + os.MkdirAll(dataDir, 0o755) + os.WriteFile(filepath.Join(dataDir, "db.bin"), []byte("x"), 0o644) + + withFakeExecutable(t, "python3", fmt.Sprintf( + "#!/bin/sh\necho x > %q\necho x > %q\n", + filepath.Join(dir, "settings.json"), filepath.Join(dir, "principals.json"), + )) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("script")) + })) + defer srv.Close() + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatal(err) + } + rs.Topology = checkpoint.Topology{StoreBackend: "rocksdb"} + + opts := Options{ + BinaryPath: binaryPath, + SkipBinaryPreservation: true, + DataDir: dataDir, + BackupDir: filepath.Join(dir, "data-backup"), + MigrationScriptURL: srv.URL, + ScriptDestPath: filepath.Join(dir, "migrate_v016.py"), + AdminURL: "https://mail.example.com", + SettingsDumpPath: filepath.Join(dir, "settings.json"), + PrincipalsDumpPath: filepath.Join(dir, "principals.json"), + } + + report, err := Run(context.Background(), store, rs, opts) + if err != nil { + t.Fatalf("Run: %v", err) + } + for _, r := range report.Results { + if r.Name == "preserve-binary" && r.Status != StatusSkipped { + t.Errorf("preserve-binary status = %s, want skipped", r.Status) + } + } + if _, err := os.Stat(binaryPath); err != nil { + t.Errorf("production binary at %s should be untouched, but stat failed: %v", binaryPath, err) + } + if _, ok := rs.Artifacts["old-binary"]; ok { + t.Error("no old-binary artifact should be recorded when preservation is skipped") + } +} diff --git a/internal/backup/binary.go b/internal/backup/binary.go new file mode 100644 index 0000000..8129012 --- /dev/null +++ b/internal/backup/binary.go @@ -0,0 +1,31 @@ +package backup + +import ( + "fmt" + "os" +) + +// PreserveBinary moves the currently-installed binary aside to +// ".v" so rollback can restart the exact old +// binary without re-downloading anything, and cutover can install the new +// one at the original path. It never deletes the old binary, and it's +// idempotent: if a prior attempt at this run already preserved it, calling +// this again just returns the existing preserved path rather than erroring +// on a missing source file. +func PreserveBinary(binaryPath, sourceVersion string) (preservedPath string, err error) { + if sourceVersion == "" { + return "", fmt.Errorf("backup: cannot preserve %s without a source version to suffix it with", binaryPath) + } + preservedPath = binaryPath + ".v" + sourceVersion + + if _, statErr := os.Stat(preservedPath); statErr == nil { + return preservedPath, nil + } else if !os.IsNotExist(statErr) { + return "", fmt.Errorf("backup: stat %s: %w", preservedPath, statErr) + } + + if err := os.Rename(binaryPath, preservedPath); err != nil { + return "", fmt.Errorf("backup: preserve %s as %s: %w", binaryPath, preservedPath, err) + } + return preservedPath, nil +} diff --git a/internal/backup/binary_test.go b/internal/backup/binary_test.go new file mode 100644 index 0000000..2cad739 --- /dev/null +++ b/internal/backup/binary_test.go @@ -0,0 +1,63 @@ +package backup + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPreserveBinary(t *testing.T) { + dir := t.TempDir() + binaryPath := filepath.Join(dir, "stalwart") + if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil { + t.Fatal(err) + } + + preserved, err := PreserveBinary(binaryPath, "0.15.5") + if err != nil { + t.Fatalf("PreserveBinary: %v", err) + } + wantPath := binaryPath + ".v0.15.5" + if preserved != wantPath { + t.Errorf("preserved = %s, want %s", preserved, wantPath) + } + if _, err := os.Stat(binaryPath); !os.IsNotExist(err) { + t.Error("original binary path should no longer exist after preservation") + } + if _, err := os.Stat(preserved); err != nil { + t.Errorf("preserved binary missing: %v", err) + } +} + +func TestPreserveBinaryIsIdempotent(t *testing.T) { + dir := t.TempDir() + binaryPath := filepath.Join(dir, "stalwart") + if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil { + t.Fatal(err) + } + + first, err := PreserveBinary(binaryPath, "0.15.5") + if err != nil { + t.Fatalf("PreserveBinary #1: %v", err) + } + + // Simulate a resumed run: the binary is already gone from binaryPath + // (moved on the prior attempt). Calling again must not error just + // because the source no longer exists. + second, err := PreserveBinary(binaryPath, "0.15.5") + if err != nil { + t.Fatalf("PreserveBinary #2 (resume): %v", err) + } + if first != second { + t.Errorf("preserved paths differ across calls: %s vs %s", first, second) + } +} + +func TestPreserveBinaryRequiresSourceVersion(t *testing.T) { + dir := t.TempDir() + binaryPath := filepath.Join(dir, "stalwart") + os.WriteFile(binaryPath, []byte("x"), 0o755) + if _, err := PreserveBinary(binaryPath, ""); err == nil { + t.Fatal("PreserveBinary with an empty source version should error") + } +} diff --git a/internal/backup/doc.go b/internal/backup/doc.go new file mode 100644 index 0000000..1fc3161 --- /dev/null +++ b/internal/backup/doc.go @@ -0,0 +1,3 @@ +// Package backup implements the defense-in-depth backup: filesystem/DB snapshot, settings dump, and Vandelay content export. +// See ARCHITECTURE.md §4.2 for the design. +package backup diff --git a/internal/backup/exectest_test.go b/internal/backup/exectest_test.go new file mode 100644 index 0000000..7cc5cbe --- /dev/null +++ b/internal/backup/exectest_test.go @@ -0,0 +1,48 @@ +package backup + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// withFakeExecutable puts a fake executable named `name` at the front of +// PATH for the duration of the test, so code that shells out to a +// real-world tool (pg_dump, mysqldump, fdbbackup, vandelay, python3) can be +// exercised without that tool actually being installed. t.Setenv restores +// PATH automatically and marks the test non-parallel. +func withFakeExecutable(t *testing.T, name, script string) (dir string) { + t.Helper() + dir = t.TempDir() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + return dir +} + +// argsFile is a shared convention the fake scripts below use: they append +// their own argv (space-joined, one invocation per line) to a file so the +// test can assert on exactly what was passed. +func argsFile(t *testing.T, dir string) string { + t.Helper() + return filepath.Join(dir, "invoked-args.log") +} + +func readArgsFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return "" + } + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func fakeScriptLoggingArgs(logPath string, extraBody string) string { + return fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\n%s\n", logPath, extraBody) +} diff --git a/internal/backup/fdbbackup.go b/internal/backup/fdbbackup.go new file mode 100644 index 0000000..1e6529c --- /dev/null +++ b/internal/backup/fdbbackup.go @@ -0,0 +1,63 @@ +package backup + +import ( + "context" + "fmt" + "os/exec" +) + +// FDBOptions configures a FoundationDB backup via fdbbackup, FDB's own +// backup CLI (not something Stalwart-specific) - see ARCHITECTURE.md §4.2. +type FDBOptions struct { + ClusterFile string // -C; empty uses fdbbackup's own default cluster file + Destination string // -d, a backup URL e.g. "file:///var/backups/stalwart-fdb" + Tag string // -t; defaults to "default" if empty +} + +func (o FDBOptions) tag() string { + if o.Tag == "" { + return "default" + } + return o.Tag +} + +// BuildFDBBackupStartArgs returns the fdbbackup argv for starting a backup, +// without the leading "fdbbackup". +func BuildFDBBackupStartArgs(o FDBOptions) []string { + args := []string{"start"} + if o.ClusterFile != "" { + args = append(args, "-C", o.ClusterFile) + } + args = append(args, "-d", o.Destination, "-t", o.tag()) + return args +} + +// StartFDBBackup kicks off an fdbbackup run. fdbbackup start returns as soon +// as the backup job is registered, not when it finishes - callers that need +// to know it's done should poll FDBBackupStatus and look for their own +// definition of "complete" in its output, since this wrapper doesn't parse +// fdbbackup's status text (that format isn't stable enough here to depend +// on without verifying it against the fdbbackup version actually in use). +func StartFDBBackup(ctx context.Context, o FDBOptions) error { + cmd := exec.CommandContext(ctx, "fdbbackup", BuildFDBBackupStartArgs(o)...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("backup: fdbbackup start failed: %w (output: %s)", err, out) + } + return nil +} + +// FDBBackupStatus returns fdbbackup's raw status output for the given tag, +// for a human or a caller-supplied parser to interpret. +func FDBBackupStatus(ctx context.Context, o FDBOptions) (string, error) { + args := []string{"status", "-t", o.tag()} + if o.ClusterFile != "" { + args = append(args, "-C", o.ClusterFile) + } + cmd := exec.CommandContext(ctx, "fdbbackup", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("backup: fdbbackup status failed: %w (output: %s)", err, out) + } + return string(out), nil +} diff --git a/internal/backup/fdbbackup_test.go b/internal/backup/fdbbackup_test.go new file mode 100644 index 0000000..09473d6 --- /dev/null +++ b/internal/backup/fdbbackup_test.go @@ -0,0 +1,48 @@ +package backup + +import ( + "context" + "strings" + "testing" +) + +func TestBuildFDBBackupStartArgs(t *testing.T) { + args := BuildFDBBackupStartArgs(FDBOptions{ClusterFile: "/etc/foundationdb/fdb.cluster", Destination: "file:///var/backups/stalwart-fdb", Tag: "stalwart-migrate"}) + joined := strings.Join(args, " ") + for _, want := range []string{"start", "-C /etc/foundationdb/fdb.cluster", "-d file:///var/backups/stalwart-fdb", "-t stalwart-migrate"} { + if !strings.Contains(joined, want) { + t.Errorf("BuildFDBBackupStartArgs = %q, missing %q", joined, want) + } + } +} + +func TestBuildFDBBackupStartArgsDefaultTag(t *testing.T) { + args := BuildFDBBackupStartArgs(FDBOptions{Destination: "file:///var/backups/stalwart-fdb"}) + joined := strings.Join(args, " ") + if !strings.Contains(joined, "-t default") { + t.Errorf("BuildFDBBackupStartArgs = %q, want default tag when none given", joined) + } +} + +func TestStartFDBBackupInvokesFdbbackup(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + withFakeExecutable(t, "fdbbackup", fakeScriptLoggingArgs(log, "exit 0")) + + err := StartFDBBackup(context.Background(), FDBOptions{Destination: "file:///var/backups/stalwart-fdb"}) + if err != nil { + t.Fatalf("StartFDBBackup: %v", err) + } + got := readArgsFile(t, log) + if !strings.Contains(got, "start") || !strings.Contains(got, "-d file:///var/backups/stalwart-fdb") { + t.Errorf("fdbbackup was invoked with %q", got) + } +} + +func TestStartFDBBackupPropagatesFailure(t *testing.T) { + withFakeExecutable(t, "fdbbackup", "#!/bin/sh\necho 'cluster unreachable' >&2\nexit 1\n") + err := StartFDBBackup(context.Background(), FDBOptions{Destination: "file:///var/backups/stalwart-fdb"}) + if err == nil { + t.Fatal("StartFDBBackup should error when fdbbackup exits non-zero") + } +} diff --git a/internal/backup/fsbackup.go b/internal/backup/fsbackup.go new file mode 100644 index 0000000..0734a2f --- /dev/null +++ b/internal/backup/fsbackup.go @@ -0,0 +1,205 @@ +package backup + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// Manifest records the per-file checksums produced by CopyDataDir, so a +// later Verify pass can detect corruption or truncation introduced by the +// copy itself. It is NOT authentication that the copied store is a valid, +// openable Stalwart database - confirming that would mean booting the old +// binary read-only against the backup, which needs config/CLI details this +// tool doesn't verify yet (see the parallel caveat on +// stalwartapi.Client.AccountSnapshot). Treat a clean Verify as "the bytes we +// wrote match the bytes we copied", not "Stalwart can open this". +type Manifest struct { + SourceDir string `json:"source_dir"` + Files []ManifestEntry `json:"files"` + TotalBytes int64 `json:"total_bytes"` +} + +type ManifestEntry struct { + RelPath string `json:"rel_path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +// Checksum returns a single content hash identifying this manifest (and +// transitively, the exact set of files and bytes it describes), suitable +// for recording as a checkpoint.Artifact's SHA256 - a directory doesn't +// have one natural hash, so this stands in for it. +func (m *Manifest) Checksum() (string, error) { + data, err := json.Marshal(m) + if err != nil { + return "", fmt.Errorf("backup: marshal manifest: %w", err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// WriteManifest persists a manifest as JSON next to the backup it describes. +func WriteManifest(path string, m *Manifest) error { + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("backup: marshal manifest: %w", err) + } + if err := os.WriteFile(path, data, 0o640); err != nil { + return fmt.Errorf("backup: write manifest %s: %w", path, err) + } + return nil +} + +// ReadManifest loads a manifest previously written by WriteManifest. +func ReadManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("backup: read manifest %s: %w", path, err) + } + var m Manifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("backup: parse manifest %s: %w", path, err) + } + return &m, nil +} + +// CopyDataDir copies srcDir to dstDir the way `cp -a` would (directories, +// regular files, and symlinks, preserving regular-file permissions), +// hashing every regular file as it's written. dstDir is cleared first if it +// already exists, so a retried step after a partial failure produces a +// clean copy rather than a mix of old and new files - filesystem copies +// aren't resumable at the byte level in any way worth building here, so a +// retry just redoes the whole thing. +func CopyDataDir(srcDir, dstDir string) (*Manifest, error) { + srcAbs, err := filepath.Abs(srcDir) + if err != nil { + return nil, fmt.Errorf("backup: resolve source %s: %w", srcDir, err) + } + dstAbs, err := filepath.Abs(dstDir) + if err != nil { + return nil, fmt.Errorf("backup: resolve destination %s: %w", dstDir, err) + } + if srcAbs == dstAbs { + return nil, fmt.Errorf("backup: source and destination are the same path: %s", srcAbs) + } + if strings.HasPrefix(dstAbs+string(filepath.Separator), srcAbs+string(filepath.Separator)) { + return nil, fmt.Errorf("backup: destination %s is inside source %s - refusing to copy a directory into itself", dstAbs, srcAbs) + } + if err := os.RemoveAll(dstAbs); err != nil { + return nil, fmt.Errorf("backup: clear stale destination %s: %w", dstAbs, err) + } + if err := os.MkdirAll(dstAbs, 0o750); err != nil { + return nil, fmt.Errorf("backup: create destination %s: %w", dstAbs, err) + } + + manifest := &Manifest{SourceDir: srcAbs} + walkErr := filepath.WalkDir(srcAbs, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcAbs, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + dstPath := filepath.Join(dstAbs, rel) + + if d.Type()&fs.ModeSymlink != 0 { + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("readlink %s: %w", path, err) + } + return os.Symlink(target, dstPath) + } + if d.IsDir() { + info, err := d.Info() + if err != nil { + return err + } + return os.MkdirAll(dstPath, info.Mode().Perm()) + } + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("backup: unsupported file type at %s (mode %s)", path, info.Mode()) + } + sum, size, err := copyFileWithChecksum(path, dstPath, info.Mode().Perm()) + if err != nil { + return fmt.Errorf("copy %s: %w", path, err) + } + manifest.Files = append(manifest.Files, ManifestEntry{RelPath: rel, SHA256: sum, Size: size}) + manifest.TotalBytes += size + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("backup: copy %s to %s: %w", srcAbs, dstAbs, walkErr) + } + return manifest, nil +} + +func copyFileWithChecksum(src, dst string, perm fs.FileMode) (sha256Hex string, size int64, err error) { + in, err := os.Open(src) + if err != nil { + return "", 0, err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return "", 0, err + } + defer out.Close() + h := sha256.New() + n, err := io.Copy(io.MultiWriter(out, h), in) + if err != nil { + return "", 0, err + } + if err := out.Sync(); err != nil { + return "", 0, err + } + return hex.EncodeToString(h.Sum(nil)), n, nil +} + +// VerifyDataDirBackup recomputes every file's checksum under dstDir and +// compares it against the manifest CopyDataDir produced, returning a single +// error describing every mismatch found (not just the first) so a human +// sees the full extent of any corruption in one pass. +func VerifyDataDirBackup(dstDir string, m *Manifest) error { + var problems []string + for _, entry := range m.Files { + path := filepath.Join(dstDir, entry.RelPath) + f, err := os.Open(path) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v", entry.RelPath, err)) + continue + } + h := sha256.New() + size, err := io.Copy(h, f) + f.Close() + if err != nil { + problems = append(problems, fmt.Sprintf("%s: read error: %v", entry.RelPath, err)) + continue + } + if size != entry.Size { + problems = append(problems, fmt.Sprintf("%s: size %d, want %d", entry.RelPath, size, entry.Size)) + continue + } + if got := hex.EncodeToString(h.Sum(nil)); got != entry.SHA256 { + problems = append(problems, fmt.Sprintf("%s: sha256 %s, want %s", entry.RelPath, got, entry.SHA256)) + } + } + if len(problems) > 0 { + return fmt.Errorf("backup: verification failed for %d of %d file(s):\n%s", len(problems), len(m.Files), strings.Join(problems, "\n")) + } + return nil +} diff --git a/internal/backup/fsbackup_test.go b/internal/backup/fsbackup_test.go new file mode 100644 index 0000000..d11a34d --- /dev/null +++ b/internal/backup/fsbackup_test.go @@ -0,0 +1,168 @@ +package backup + +import ( + "os" + "path/filepath" + "testing" +) + +func writeTree(t *testing.T, root string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "a.db"), []byte("alpha data"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "sub", "b.db"), []byte("bravo data, a bit longer"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink("a.db", filepath.Join(root, "link-to-a")); err != nil { + t.Fatal(err) + } +} + +func TestCopyDataDirAndVerify(t *testing.T) { + src := filepath.Join(t.TempDir(), "data") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + writeTree(t, src) + + dst := filepath.Join(t.TempDir(), "data-backup") + manifest, err := CopyDataDir(src, dst) + if err != nil { + t.Fatalf("CopyDataDir: %v", err) + } + if len(manifest.Files) != 2 { + t.Fatalf("manifest has %d files, want 2 (symlinks aren't hashed): %+v", len(manifest.Files), manifest.Files) + } + if manifest.TotalBytes != int64(len("alpha data")+len("bravo data, a bit longer")) { + t.Errorf("TotalBytes = %d, want %d", manifest.TotalBytes, len("alpha data")+len("bravo data, a bit longer")) + } + + // The copy should be byte-identical, including the symlink. + got, err := os.ReadFile(filepath.Join(dst, "sub", "b.db")) + if err != nil || string(got) != "bravo data, a bit longer" { + t.Errorf("copied sub/b.db = %q, %v", got, err) + } + target, err := os.Readlink(filepath.Join(dst, "link-to-a")) + if err != nil || target != "a.db" { + t.Errorf("copied symlink target = %q, %v, want a.db", target, err) + } + + if err := VerifyDataDirBackup(dst, manifest); err != nil { + t.Errorf("VerifyDataDirBackup on an untouched copy: %v", err) + } +} + +func TestCopyDataDirRefusesSelfCopy(t *testing.T) { + dir := filepath.Join(t.TempDir(), "data") + os.MkdirAll(dir, 0o755) + if _, err := CopyDataDir(dir, dir); err == nil { + t.Fatal("CopyDataDir(dir, dir) should refuse to copy a directory onto itself") + } +} + +func TestCopyDataDirRefusesNestedDestination(t *testing.T) { + src := filepath.Join(t.TempDir(), "data") + os.MkdirAll(src, 0o755) + nested := filepath.Join(src, "backup") + if _, err := CopyDataDir(src, nested); err == nil { + t.Fatal("CopyDataDir should refuse a destination nested inside the source") + } +} + +func TestCopyDataDirRetryClearsStaleFiles(t *testing.T) { + src := filepath.Join(t.TempDir(), "data") + os.MkdirAll(src, 0o755) + writeTree(t, src) + + dst := filepath.Join(t.TempDir(), "data-backup") + // Simulate a stale partial copy from a previous failed attempt. + os.MkdirAll(dst, 0o755) + if err := os.WriteFile(filepath.Join(dst, "stale-leftover.tmp"), []byte("junk"), 0o644); err != nil { + t.Fatal(err) + } + + manifest, err := CopyDataDir(src, dst) + if err != nil { + t.Fatalf("CopyDataDir: %v", err) + } + if _, err := os.Stat(filepath.Join(dst, "stale-leftover.tmp")); !os.IsNotExist(err) { + t.Error("stale-leftover.tmp should have been cleared by a fresh copy, but still exists") + } + if err := VerifyDataDirBackup(dst, manifest); err != nil { + t.Errorf("VerifyDataDirBackup after retry: %v", err) + } +} + +func TestVerifyDataDirBackupDetectsTampering(t *testing.T) { + src := filepath.Join(t.TempDir(), "data") + os.MkdirAll(src, 0o755) + writeTree(t, src) + + dst := filepath.Join(t.TempDir(), "data-backup") + manifest, err := CopyDataDir(src, dst) + if err != nil { + t.Fatalf("CopyDataDir: %v", err) + } + + // Corrupt the backup after the fact - Verify must catch it. + if err := os.WriteFile(filepath.Join(dst, "a.db"), []byte("corrupted!"), 0o644); err != nil { + t.Fatal(err) + } + + if err := VerifyDataDirBackup(dst, manifest); err == nil { + t.Fatal("VerifyDataDirBackup should have detected the tampered file") + } +} + +func TestManifestChecksumIsDeterministic(t *testing.T) { + src := filepath.Join(t.TempDir(), "data") + os.MkdirAll(src, 0o755) + writeTree(t, src) + + dst := filepath.Join(t.TempDir(), "data-backup") + m1, err := CopyDataDir(src, dst) + if err != nil { + t.Fatal(err) + } + sum1, err := m1.Checksum() + if err != nil { + t.Fatal(err) + } + + dst2 := filepath.Join(t.TempDir(), "data-backup-2") + m2, err := CopyDataDir(src, dst2) + if err != nil { + t.Fatal(err) + } + sum2, err := m2.Checksum() + if err != nil { + t.Fatal(err) + } + + if sum1 != sum2 { + t.Errorf("two copies of the same source produced different manifest checksums: %s vs %s", sum1, sum2) + } +} + +func TestWriteReadManifestRoundtrip(t *testing.T) { + m := &Manifest{ + SourceDir: "/var/lib/stalwart", + Files: []ManifestEntry{{RelPath: "a.db", SHA256: "deadbeef", Size: 42}}, + TotalBytes: 42, + } + path := filepath.Join(t.TempDir(), "manifest.json") + if err := WriteManifest(path, m); err != nil { + t.Fatalf("WriteManifest: %v", err) + } + got, err := ReadManifest(path) + if err != nil { + t.Fatalf("ReadManifest: %v", err) + } + if got.TotalBytes != 42 || len(got.Files) != 1 || got.Files[0].SHA256 != "deadbeef" { + t.Errorf("roundtrip mismatch: %+v", got) + } +} diff --git a/internal/backup/hash.go b/internal/backup/hash.go new file mode 100644 index 0000000..ec22ec7 --- /dev/null +++ b/internal/backup/hash.go @@ -0,0 +1,25 @@ +package backup + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" +) + +// hashFile returns a file's SHA256 and size, for recording as a +// checkpoint.Artifact. +func hashFile(path string) (sha256Hex string, size int64, err error) { + f, err := os.Open(path) + if err != nil { + return "", 0, fmt.Errorf("backup: hash %s: %w", path, err) + } + defer f.Close() + h := sha256.New() + n, err := io.Copy(h, f) + if err != nil { + return "", 0, fmt.Errorf("backup: hash %s: %w", path, err) + } + return hex.EncodeToString(h.Sum(nil)), n, nil +} diff --git a/internal/backup/report.go b/internal/backup/report.go new file mode 100644 index 0000000..c111865 --- /dev/null +++ b/internal/backup/report.go @@ -0,0 +1,36 @@ +package backup + +import ( + "fmt" + "strings" +) + +// Status is a single backup step's verdict. Unlike preflight, most backup +// steps that fail are hard failures (Run aborts) rather than advisory - see +// Run's doc comment - but Status still distinguishes "did the thing" from +// "correctly skipped because it doesn't apply to this deployment". +type Status string + +const ( + StatusOK Status = "ok" + StatusSkipped Status = "skipped" + StatusFail Status = "fail" +) + +type CheckResult struct { + Name string + Status Status + Detail string +} + +type Report struct { + Results []CheckResult +} + +func (r Report) String() string { + var b strings.Builder + for _, res := range r.Results { + fmt.Fprintf(&b, "[%-7s] %-18s %s\n", strings.ToUpper(string(res.Status)), res.Name, res.Detail) + } + return b.String() +} diff --git a/internal/backup/settingsdump.go b/internal/backup/settingsdump.go new file mode 100644 index 0000000..bccbb61 --- /dev/null +++ b/internal/backup/settingsdump.go @@ -0,0 +1,164 @@ +package backup + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "sort" + "strings" + "time" +) + +// DefaultMigrationScriptURL is Stalwart's own v0.15->v0.16 settings +// converter, referenced directly from UPGRADING/v0_16.md. It's an external, +// Stalwart-owned dependency this tool doesn't vendor a copy of - see the +// pinning discussion on DownloadFile and ARCHITECTURE.md §8. +const DefaultMigrationScriptURL = "https://raw.githubusercontent.com/stalwartlabs/stalwart/main/resources/scripts/migrate_v016.py" + +// DownloadFile fetches url to destPath and returns its SHA256. If +// expectedSHA256 is non-empty, a mismatching download is rejected (and the +// partial file removed) - this is how a pinned migration-script hash is +// enforced, so a run never silently executes a different version of a +// script than the one it was reviewed against. If expectedSHA256 is empty, +// the download is accepted unconditionally and its hash is returned so the +// caller can record it as the pin for next time; first-run trust-on-first-use +// is a known gap, flagged in ARCHITECTURE.md §8. +func DownloadFile(ctx context.Context, httpClient *http.Client, url, destPath, expectedSHA256 string) (sha256Hex string, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + if httpClient == nil { + httpClient = &http.Client{Timeout: 60 * time.Second} + } + resp, err := httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("backup: fetch %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("backup: fetch %s: unexpected status %s", url, resp.Status) + } + + f, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) + if err != nil { + return "", fmt.Errorf("backup: create %s: %w", destPath, err) + } + h := sha256.New() + _, copyErr := io.Copy(io.MultiWriter(f, h), resp.Body) + closeErr := f.Close() + if copyErr != nil { + os.Remove(destPath) + return "", fmt.Errorf("backup: download %s: %w", url, copyErr) + } + if closeErr != nil { + os.Remove(destPath) + return "", fmt.Errorf("backup: close %s: %w", destPath, closeErr) + } + + sha256Hex = hex.EncodeToString(h.Sum(nil)) + if expectedSHA256 != "" && sha256Hex != expectedSHA256 { + os.Remove(destPath) + return "", fmt.Errorf( + "backup: %s checksum mismatch: got %s, want %s (refusing to run an unexpected version of a script that irreversibly wipes settings on first v0.16 start)", + url, sha256Hex, expectedSHA256, + ) + } + return sha256Hex, nil +} + +// SettingsDumpOptions configures a migrate_v016.py `dump` invocation +// against a live v0.15.x instance - see UPGRADING/v0_16.md. +type SettingsDumpOptions struct { + PythonPath string // defaults to "python3" + ScriptPath string // local path to the already-downloaded, checksum-verified script + URL string // the running v0.15.x instance's base URL + Username string + Password string + SettingsPath string + PrincipalsPath string +} + +// RunSettingsDump runs migrate_v016.py's dump subcommand, which reads the +// live v0.15.x server's settings and principals over its admin API and +// writes them to SettingsPath/PrincipalsPath for the later convert step +// (ARCHITECTURE.md §4.3). This step is read-only against the server, so +// it's safe to run well before cutover - ARCHITECTURE.md §4.2 calls for it +// both at preflight time and again immediately before cutover, since the +// live settings may have changed in between. +func RunSettingsDump(ctx context.Context, o SettingsDumpOptions) error { + python := o.PythonPath + if python == "" { + python = "python3" + } + args := []string{ + o.ScriptPath, "dump", + "--url", o.URL, + "--username", o.Username, + "--password", o.Password, + "--settings", o.SettingsPath, + "--principals", o.PrincipalsPath, + } + cmd := exec.CommandContext(ctx, python, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("backup: migrate_v016.py dump failed: %w (output: %s)", err, out) + } + return nil +} + +// SettingsConvertOptions configures a migrate_v016.py `convert` invocation, +// which turns the settings/principals dump into the v0.16 config.json and +// export.json that recovery mode consumes - see UPGRADING/v0_16.md. +type SettingsConvertOptions struct { + PythonPath string // defaults to "python3" + ScriptPath string + SettingsPath string + PrincipalsPath string + ConfigPath string // output: config.json for the new binary's --config flag + OutputPath string // output: export.json for `stalwart-cli apply` + + // PatchPaths rewrites path prefixes in the generated config (documented + // for Docker deployments as "--patch-paths /opt/stalwart=/var/lib/stalwart", + // e.g. old-path -> new-path). This is the officially documented + // mechanism a dry-run relies on to point the generated config at a + // sandbox data directory instead of the production one - see + // ARCHITECTURE.md's dry-run design - rather than this tool editing + // config.json's contents directly, which would require depending on its + // exact schema. + PatchPaths map[string]string +} + +// RunSettingsConvert runs migrate_v016.py's convert subcommand. +func RunSettingsConvert(ctx context.Context, o SettingsConvertOptions) error { + python := o.PythonPath + if python == "" { + python = "python3" + } + args := []string{ + o.ScriptPath, "convert", + "--settings", o.SettingsPath, + "--principals", o.PrincipalsPath, + "--config", o.ConfigPath, + "--output", o.OutputPath, + } + if len(o.PatchPaths) > 0 { + pairs := make([]string, 0, len(o.PatchPaths)) + for old, new := range o.PatchPaths { + pairs = append(pairs, old+"="+new) + } + sort.Strings(pairs) // deterministic argv, easier to test and to log + args = append(args, "--patch-paths", strings.Join(pairs, ",")) + } + cmd := exec.CommandContext(ctx, python, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("backup: migrate_v016.py convert failed: %w (output: %s)", err, out) + } + return nil +} diff --git a/internal/backup/settingsdump_test.go b/internal/backup/settingsdump_test.go new file mode 100644 index 0000000..9dcc442 --- /dev/null +++ b/internal/backup/settingsdump_test.go @@ -0,0 +1,175 @@ +package backup + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDownloadFileAcceptsMatchingChecksum(t *testing.T) { + content := "print('fake migration script')\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(content)) + })) + defer srv.Close() + + sum := sha256.Sum256([]byte(content)) + expected := hex.EncodeToString(sum[:]) + + dest := filepath.Join(t.TempDir(), "script.py") + got, err := DownloadFile(context.Background(), nil, srv.URL, dest, expected) + if err != nil { + t.Fatalf("DownloadFile: %v", err) + } + if got != expected { + t.Errorf("returned checksum = %s, want %s", got, expected) + } + data, _ := os.ReadFile(dest) + if string(data) != content { + t.Errorf("downloaded content = %q, want %q", data, content) + } +} + +func TestDownloadFileRejectsMismatchedChecksum(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("unexpected content")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "script.py") + _, err := DownloadFile(context.Background(), nil, srv.URL, dest, "0000000000000000000000000000000000000000000000000000000000000000") + if err == nil { + t.Fatal("DownloadFile should reject a checksum mismatch") + } + if _, statErr := os.Stat(dest); !os.IsNotExist(statErr) { + t.Error("DownloadFile should remove the file it wrote after a checksum mismatch") + } +} + +func TestDownloadFileWithoutPinReturnsComputedHash(t *testing.T) { + content := "arbitrary content" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(content)) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "script.py") + got, err := DownloadFile(context.Background(), nil, srv.URL, dest, "") + if err != nil { + t.Fatalf("DownloadFile: %v", err) + } + sum := sha256.Sum256([]byte(content)) + want := hex.EncodeToString(sum[:]) + if got != want { + t.Errorf("returned checksum = %s, want %s", got, want) + } +} + +func TestRunSettingsDumpInvokesScriptWithFlags(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + pythonDir := withFakeExecutable(t, "python3", fakeScriptLoggingArgs(log, "exit 0")) + + err := RunSettingsDump(context.Background(), SettingsDumpOptions{ + PythonPath: filepath.Join(pythonDir, "python3"), + ScriptPath: "/opt/migrate_v016.py", + URL: "https://mail.example.com", + Username: "admin", + Password: "hunter2", + SettingsPath: filepath.Join(dir, "settings.json"), + PrincipalsPath: filepath.Join(dir, "principals.json"), + }) + if err != nil { + t.Fatalf("RunSettingsDump: %v", err) + } + got := readArgsFile(t, log) + for _, want := range []string{"/opt/migrate_v016.py", "dump", "--url https://mail.example.com", "--username admin"} { + if !strings.Contains(got, want) { + t.Errorf("script invoked with %q, missing %q", got, want) + } + } +} + +func TestRunSettingsDumpPropagatesFailure(t *testing.T) { + pythonDir := withFakeExecutable(t, "python3", "#!/bin/sh\necho 'auth failed' >&2\nexit 1\n") + err := RunSettingsDump(context.Background(), SettingsDumpOptions{ + PythonPath: filepath.Join(pythonDir, "python3"), + ScriptPath: "/opt/migrate_v016.py", + }) + if err == nil { + t.Fatal("RunSettingsDump should error when the script exits non-zero") + } + if !strings.Contains(err.Error(), "auth failed") { + t.Errorf("error = %v, want it to include the script's stderr", err) + } +} + +func TestRunSettingsConvertInvokesScriptWithFlags(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + pythonDir := withFakeExecutable(t, "python3", fakeScriptLoggingArgs(log, "exit 0")) + + err := RunSettingsConvert(context.Background(), SettingsConvertOptions{ + PythonPath: filepath.Join(pythonDir, "python3"), + ScriptPath: "/opt/migrate_v016.py", + SettingsPath: filepath.Join(dir, "settings.json"), + PrincipalsPath: filepath.Join(dir, "principals.json"), + ConfigPath: filepath.Join(dir, "config.json"), + OutputPath: filepath.Join(dir, "export.json"), + }) + if err != nil { + t.Fatalf("RunSettingsConvert: %v", err) + } + got := readArgsFile(t, log) + for _, want := range []string{"convert", "--settings", "--config", "--output"} { + if !strings.Contains(got, want) { + t.Errorf("script invoked with %q, missing %q", got, want) + } + } + if strings.Contains(got, "--patch-paths") { + t.Errorf("script invoked with %q, should not include --patch-paths when none given", got) + } +} + +func TestRunSettingsConvertWithPatchPaths(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + pythonDir := withFakeExecutable(t, "python3", fakeScriptLoggingArgs(log, "exit 0")) + + err := RunSettingsConvert(context.Background(), SettingsConvertOptions{ + PythonPath: filepath.Join(pythonDir, "python3"), + ScriptPath: "/opt/migrate_v016.py", + SettingsPath: filepath.Join(dir, "settings.json"), + PrincipalsPath: filepath.Join(dir, "principals.json"), + ConfigPath: filepath.Join(dir, "config.json"), + OutputPath: filepath.Join(dir, "export.json"), + PatchPaths: map[string]string{"/var/lib/stalwart": "/tmp/sandbox/stalwart"}, + }) + if err != nil { + t.Fatalf("RunSettingsConvert: %v", err) + } + got := readArgsFile(t, log) + if !strings.Contains(got, "--patch-paths /var/lib/stalwart=/tmp/sandbox/stalwart") { + t.Errorf("script invoked with %q, missing the expected --patch-paths flag", got) + } +} + +func TestRunSettingsConvertPropagatesFailure(t *testing.T) { + pythonDir := withFakeExecutable(t, "python3", "#!/bin/sh\necho 'unsupported settings key' >&2\nexit 1\n") + err := RunSettingsConvert(context.Background(), SettingsConvertOptions{ + PythonPath: filepath.Join(pythonDir, "python3"), + ScriptPath: "/opt/migrate_v016.py", + }) + if err == nil { + t.Fatal("RunSettingsConvert should error when the script exits non-zero") + } + if !strings.Contains(err.Error(), "unsupported settings key") { + t.Errorf("error = %v, want it to include the script's stderr", err) + } +} diff --git a/internal/backup/sqlbackup.go b/internal/backup/sqlbackup.go new file mode 100644 index 0000000..6765a24 --- /dev/null +++ b/internal/backup/sqlbackup.go @@ -0,0 +1,91 @@ +package backup + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" +) + +// criticalTables is the exact table set Stalwart's own v0.16 upgrade guide +// backs up before migrating (principals/directory, domains, and the other +// tables its migration script and recovery mode depend on) - a targeted +// dump, not a full-instance one, matching the guide's own tested restore +// path and staying fast on large installs. See ARCHITECTURE.md §4.2 and +// UPGRADING/v0_16.md. +var criticalTables = []string{"s", "d", "r", "h", "b", "g", "j", "f", "u"} + +// SQLOptions configures a targeted critical-table dump for an external SQL +// store backend. +type SQLOptions struct { + Host string + Port string + Database string + User string + Password string + OutPath string +} + +// BuildPgDumpArgs returns pg_dump's argv for the critical-table backup, +// without the leading "pg_dump" itself. +func BuildPgDumpArgs(o SQLOptions) []string { + args := []string{"-U", o.User, "-d", o.Database} + if o.Host != "" { + args = append(args, "-h", o.Host) + } + if o.Port != "" { + args = append(args, "-p", o.Port) + } + for _, t := range criticalTables { + args = append(args, "-t", t) + } + return append(args, "-f", o.OutPath) +} + +// BuildMySQLDumpArgs returns mysqldump's argv for the same critical-table +// set. mysqldump writes to stdout, so RunMySQLDump redirects it rather than +// this function taking an output path flag. +func BuildMySQLDumpArgs(o SQLOptions) []string { + args := []string{"-u", o.User, o.Database} + if o.Host != "" { + args = append(args, "-h", o.Host) + } + if o.Port != "" { + args = append(args, "-P", o.Port) + } + return append(args, criticalTables...) +} + +// RunPgDump executes pg_dump with the password passed via the standard +// PGPASSWORD environment variable, never on the command line where it would +// be visible to anything reading this process's argv (e.g. `ps`). +func RunPgDump(ctx context.Context, o SQLOptions) error { + cmd := exec.CommandContext(ctx, "pg_dump", BuildPgDumpArgs(o)...) + cmd.Env = append(os.Environ(), "PGPASSWORD="+o.Password) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("backup: pg_dump failed: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + return nil +} + +// RunMySQLDump executes mysqldump with the password passed via the standard +// MYSQL_PWD environment variable, redirecting its stdout to o.OutPath. +func RunMySQLDump(ctx context.Context, o SQLOptions) error { + f, err := os.OpenFile(o.OutPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) + if err != nil { + return fmt.Errorf("backup: create %s: %w", o.OutPath, err) + } + defer f.Close() + + cmd := exec.CommandContext(ctx, "mysqldump", BuildMySQLDumpArgs(o)...) + cmd.Env = append(os.Environ(), "MYSQL_PWD="+o.Password) + cmd.Stdout = f + var stderr strings.Builder + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("backup: mysqldump failed: %w (stderr: %s)", err, strings.TrimSpace(stderr.String())) + } + return nil +} diff --git a/internal/backup/sqlbackup_test.go b/internal/backup/sqlbackup_test.go new file mode 100644 index 0000000..b09da0d --- /dev/null +++ b/internal/backup/sqlbackup_test.go @@ -0,0 +1,85 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestBuildPgDumpArgs(t *testing.T) { + args := BuildPgDumpArgs(SQLOptions{ + Host: "db.internal", Port: "5432", Database: "stalwart", User: "stalwart", OutPath: "/backups/out.sql", + }) + joined := strings.Join(args, " ") + for _, want := range []string{"-U stalwart", "-d stalwart", "-h db.internal", "-p 5432", "-f /backups/out.sql"} { + if !strings.Contains(joined, want) { + t.Errorf("BuildPgDumpArgs = %q, missing %q", joined, want) + } + } + for _, table := range criticalTables { + if !strings.Contains(joined, "-t "+table) { + t.Errorf("BuildPgDumpArgs missing critical table flag for %q: %q", table, joined) + } + } +} + +func TestBuildMySQLDumpArgs(t *testing.T) { + args := BuildMySQLDumpArgs(SQLOptions{Host: "db.internal", Port: "3306", Database: "stalwart", User: "stalwart"}) + joined := strings.Join(args, " ") + if !strings.Contains(joined, "-u stalwart") || !strings.Contains(joined, "-h db.internal") || !strings.Contains(joined, "-P 3306") { + t.Errorf("BuildMySQLDumpArgs = %q, missing expected flags", joined) + } + if !strings.HasSuffix(joined, strings.Join(criticalTables, " ")) { + t.Errorf("BuildMySQLDumpArgs = %q, want it to end with the critical table list", joined) + } +} + +func TestRunPgDumpInvokesPgDumpWithArgs(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + withFakeExecutable(t, "pg_dump", fakeScriptLoggingArgs(log, "exit 0")) + + err := RunPgDump(context.Background(), SQLOptions{ + User: "stalwart", Database: "stalwart", OutPath: filepath.Join(dir, "out.sql"), + }) + if err != nil { + t.Fatalf("RunPgDump: %v", err) + } + got := readArgsFile(t, log) + if !strings.Contains(got, "-U stalwart") { + t.Errorf("pg_dump was invoked with %q, missing -U stalwart", got) + } +} + +func TestRunPgDumpPropagatesFailure(t *testing.T) { + dir := t.TempDir() + withFakeExecutable(t, "pg_dump", "#!/bin/sh\necho 'connection refused' >&2\nexit 1\n") + + err := RunPgDump(context.Background(), SQLOptions{User: "stalwart", Database: "stalwart", OutPath: filepath.Join(dir, "out.sql")}) + if err == nil { + t.Fatal("RunPgDump should have returned an error when pg_dump exits non-zero") + } + if !strings.Contains(err.Error(), "connection refused") { + t.Errorf("error = %v, want it to include pg_dump's stderr", err) + } +} + +func TestRunMySQLDumpWritesStdoutToOutPath(t *testing.T) { + dir := t.TempDir() + withFakeExecutable(t, "mysqldump", "#!/bin/sh\necho '-- fake dump output'\n") + + outPath := filepath.Join(dir, "out.sql") + err := RunMySQLDump(context.Background(), SQLOptions{User: "stalwart", Database: "stalwart", OutPath: outPath}) + if err != nil { + t.Fatalf("RunMySQLDump: %v", err) + } + data, err := os.ReadFile(outPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !strings.Contains(string(data), "fake dump output") { + t.Errorf("out.sql = %q, want fake dump output redirected into it", data) + } +} diff --git a/internal/backup/vandelay.go b/internal/backup/vandelay.go new file mode 100644 index 0000000..b97ebcc --- /dev/null +++ b/internal/backup/vandelay.go @@ -0,0 +1,67 @@ +package backup + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" +) + +// VandelayOptions configures a per-account content export via Stalwart's +// own Vandelay import/export tool - the documented, backend-independent +// backup mechanism (ARCHITECTURE.md §4.2's belt-and-suspenders layer): each +// account's mail, calendars, contacts, Sieve scripts, and identities land in +// one self-contained SQLite archive. +type VandelayOptions struct { + BinaryPath string // defaults to "vandelay" + URL string // the source JMAP server + AuthBasic string // "user:app-password", per vandelay's own --auth-basic flag + OutDir string // one .sqlite file per account goes here +} + +// BuildVandelayImportArgs returns vandelay's argv for exporting one +// account's content into a self-contained SQLite archive, without the +// leading "vandelay" itself. "import" is vandelay's own verb for this - it +// names the direction relative to the archive file, not the live server; +// see ARCHITECTURE.md §4.2. +func BuildVandelayImportArgs(o VandelayOptions, accountName, outFile string) []string { + return []string{ + "import", "jmap", + "--url", o.URL, + "--auth-basic", o.AuthBasic, + "--account-name", accountName, + outFile, + } +} + +// ExportAccount runs one account's Vandelay export and returns the archive +// path it wrote. +func ExportAccount(ctx context.Context, o VandelayOptions, accountName string) (outFile string, err error) { + binary := o.BinaryPath + if binary == "" { + binary = "vandelay" + } + outFile = filepath.Join(o.OutDir, accountName+".sqlite") + cmd := exec.CommandContext(ctx, binary, BuildVandelayImportArgs(o, accountName, outFile)...) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("backup: vandelay export of %s failed: %w (output: %s)", accountName, err, out) + } + return outFile, nil +} + +// ExportAccounts exports every account in accounts, stopping at the first +// failure. A partial per-account backup set is reported precisely (how many +// succeeded, which one failed and why) rather than silently continuing past +// a failure that might indicate a systemic problem - bad credentials, an +// unreachable server - rather than a one-off. +func ExportAccounts(ctx context.Context, o VandelayOptions, accounts []string) (outFiles []string, err error) { + for _, acct := range accounts { + f, err := ExportAccount(ctx, o, acct) + if err != nil { + return outFiles, fmt.Errorf("backup: exported %d/%d accounts before failing: %w", len(outFiles), len(accounts), err) + } + outFiles = append(outFiles, f) + } + return outFiles, nil +} diff --git a/internal/backup/vandelay_test.go b/internal/backup/vandelay_test.go new file mode 100644 index 0000000..1cde05e --- /dev/null +++ b/internal/backup/vandelay_test.go @@ -0,0 +1,56 @@ +package backup + +import ( + "context" + "path/filepath" + "strings" + "testing" +) + +func TestBuildVandelayImportArgs(t *testing.T) { + args := BuildVandelayImportArgs(VandelayOptions{URL: "https://mail.example.com", AuthBasic: "alice:app-pass"}, "alice@example.com", "/backups/alice.sqlite") + joined := strings.Join(args, " ") + for _, want := range []string{"import", "jmap", "--url https://mail.example.com", "--auth-basic alice:app-pass", "--account-name alice@example.com", "/backups/alice.sqlite"} { + if !strings.Contains(joined, want) { + t.Errorf("BuildVandelayImportArgs = %q, missing %q", joined, want) + } + } +} + +func TestExportAccountsRunsEachAccount(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + withFakeExecutable(t, "vandelay", fakeScriptLoggingArgs(log, "exit 0")) + + outDir := filepath.Join(dir, "out") + files, err := ExportAccounts(context.Background(), VandelayOptions{URL: "https://mail.example.com", AuthBasic: "a:b", OutDir: outDir}, []string{"alice@example.com", "bob@example.com"}) + if err != nil { + t.Fatalf("ExportAccounts: %v", err) + } + if len(files) != 2 { + t.Fatalf("got %d files, want 2: %v", len(files), files) + } + got := readArgsFile(t, log) + if !strings.Contains(got, "alice@example.com") || !strings.Contains(got, "bob@example.com") { + t.Errorf("vandelay invocations = %q, want both accounts", got) + } +} + +func TestExportAccountsStopsAtFirstFailure(t *testing.T) { + dir := t.TempDir() + // Fails on the second invocation (bob), succeeds on the first (alice). + script := "#!/bin/sh\ncase \"$*\" in\n *bob*) echo 'account not found' >&2; exit 1 ;;\n *) exit 0 ;;\nesac\n" + withFakeExecutable(t, "vandelay", script) + + outDir := filepath.Join(dir, "out") + files, err := ExportAccounts(context.Background(), VandelayOptions{URL: "https://mail.example.com", AuthBasic: "a:b", OutDir: outDir}, []string{"alice@example.com", "bob@example.com", "carol@example.com"}) + if err == nil { + t.Fatal("ExportAccounts should have failed on bob@example.com") + } + if len(files) != 1 { + t.Errorf("got %d successful exports before failure, want 1 (alice only)", len(files)) + } + if !strings.Contains(err.Error(), "1/3") { + t.Errorf("error = %v, want it to report 1/3 accounts exported before failing", err) + } +} diff --git a/internal/checkpoint/doc.go b/internal/checkpoint/doc.go new file mode 100644 index 0000000..239c28e --- /dev/null +++ b/internal/checkpoint/doc.go @@ -0,0 +1,3 @@ +// Package checkpoint implements run-id and state.json persistence, resume logic, and rollback-window tracking. +// See ARCHITECTURE.md §5 for the design. +package checkpoint diff --git a/internal/checkpoint/step.go b/internal/checkpoint/step.go new file mode 100644 index 0000000..123e17b --- /dev/null +++ b/internal/checkpoint/step.go @@ -0,0 +1,88 @@ +package checkpoint + +import ( + "fmt" + "time" +) + +func (rs *RunState) stepIndex(phase Phase, name string) int { + for i := range rs.Steps { + if rs.Steps[i].Phase == phase && rs.Steps[i].Name == name { + return i + } + } + return -1 +} + +// Status returns the current status of a step, or StepPending if it has +// never been started. +func (rs *RunState) Status(phase Phase, name string) StepStatus { + if i := rs.stepIndex(phase, name); i >= 0 { + return rs.Steps[i].Status + } + return StepPending +} + +// Done reports whether a step already completed successfully. Callers use +// this to decide whether to skip work on resume. +func (rs *RunState) Done(phase Phase, name string) bool { + return rs.Status(phase, name) == StepDone +} + +// Outcome returns the StepOutcome recorded for a step, zero-valued if none. +// This is what lets a resumed run reconstruct a skipped step's result +// without re-executing it. +func (rs *RunState) Outcome(phase Phase, name string) StepOutcome { + if i := rs.stepIndex(phase, name); i >= 0 { + return rs.Steps[i].StepOutcome + } + return StepOutcome{} +} + +// Begin marks a step as running, creating its record on first attempt or +// resetting it on a retry after a prior failure. +func (rs *RunState) Begin(phase Phase, name string) { + now := time.Now().UTC() + if i := rs.stepIndex(phase, name); i >= 0 { + rs.Steps[i].Status = StepRunning + rs.Steps[i].StartedAt = &now + rs.Steps[i].CompletedAt = nil + rs.Steps[i].StepOutcome = StepOutcome{} + rs.Steps[i].Error = "" + } else { + rs.Steps = append(rs.Steps, StepRecord{ + Phase: phase, Name: name, Status: StepRunning, StartedAt: &now, + }) + } + rs.UpdatedAt = now +} + +// Complete marks a step done with its outcome. It panics if Begin was never +// called for this step - that's a bug in the calling phase, not a runtime +// condition callers should need to handle. +func (rs *RunState) Complete(phase Phase, name string, outcome StepOutcome) { + i := rs.stepIndex(phase, name) + if i < 0 { + panic(fmt.Sprintf("checkpoint: Complete(%s/%s) called without Begin", phase, name)) + } + now := time.Now().UTC() + rs.Steps[i].Status = StepDone + rs.Steps[i].CompletedAt = &now + rs.Steps[i].StepOutcome = outcome + rs.Steps[i].Error = "" + rs.UpdatedAt = now +} + +// Fail marks a step failed, so a later Begin for the same (phase, name) +// knows to retry it rather than treat it as done. +func (rs *RunState) Fail(phase Phase, name string, stepErr error) { + i := rs.stepIndex(phase, name) + if i < 0 { + panic(fmt.Sprintf("checkpoint: Fail(%s/%s) called without Begin", phase, name)) + } + now := time.Now().UTC() + rs.Steps[i].Status = StepFailed + rs.Steps[i].CompletedAt = &now + rs.Steps[i].Error = stepErr.Error() + rs.UpdatedAt = now +} diff --git a/internal/checkpoint/store.go b/internal/checkpoint/store.go new file mode 100644 index 0000000..30182cd --- /dev/null +++ b/internal/checkpoint/store.go @@ -0,0 +1,179 @@ +package checkpoint + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "time" +) + +// DefaultBaseDir is where runs are persisted when the operator doesn't +// override it. +const DefaultBaseDir = "/var/lib/stalwart-migrator/runs" + +// Store persists RunState to disk. It's the only thing in this package that +// touches the filesystem - RunState itself is a plain data type. +type Store struct { + baseDir string +} + +func NewStore(baseDir string) *Store { + if baseDir == "" { + baseDir = DefaultBaseDir + } + return &Store{baseDir: baseDir} +} + +func (s *Store) runDir(runID string) string { return filepath.Join(s.baseDir, runID) } +func (s *Store) statePath(runID string) string { return filepath.Join(s.runDir(runID), "state.json") } + +// Create starts a new run, assigns it an ID, and persists its initial +// state before returning it. +func (s *Store) Create(sourceVersion, targetVersion string) (*RunState, error) { + id, err := newRunID() + if err != nil { + return nil, fmt.Errorf("checkpoint: generate run id: %w", err) + } + now := time.Now().UTC() + rs := &RunState{ + RunID: id, + SourceVersion: sourceVersion, + TargetVersion: targetVersion, + CreatedAt: now, + UpdatedAt: now, + Artifacts: map[string]Artifact{}, + } + if err := s.Save(rs); err != nil { + return nil, err + } + return rs, nil +} + +// Load reads an existing run's state from disk. +func (s *Store) Load(runID string) (*RunState, error) { + data, err := os.ReadFile(s.statePath(runID)) + if err != nil { + return nil, fmt.Errorf("checkpoint: load run %s: %w", runID, err) + } + var rs RunState + if err := json.Unmarshal(data, &rs); err != nil { + return nil, fmt.Errorf("checkpoint: parse run %s: %w", runID, err) + } + return &rs, nil +} + +// Save writes state to disk atomically: write to a temp file in the same +// directory, fsync it, then rename over the real path. A crash mid-write +// leaves the temp file orphaned and state.json untouched, never a +// truncated or corrupt state.json - that file is the one thing every phase +// and a human operator both trust as the source of truth for what's +// already happened, so a half-written version of it would be worse than an +// old one. +func (s *Store) Save(rs *RunState) error { + dir := s.runDir(rs.RunID) + if err := os.MkdirAll(dir, 0o750); err != nil { + return fmt.Errorf("checkpoint: create run directory: %w", err) + } + data, err := json.MarshalIndent(rs, "", " ") + if err != nil { + return fmt.Errorf("checkpoint: marshal state: %w", err) + } + tmp, err := os.CreateTemp(dir, "state-*.json.tmp") + if err != nil { + return fmt.Errorf("checkpoint: create temp state file: %w", err) + } + tmpPath := tmp.Name() + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmpPath) + return fmt.Errorf("checkpoint: write temp state file: %w", err) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + os.Remove(tmpPath) + return fmt.Errorf("checkpoint: sync temp state file: %w", err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("checkpoint: close temp state file: %w", err) + } + if err := os.Rename(tmpPath, s.statePath(rs.RunID)); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("checkpoint: rename temp state file into place: %w", err) + } + return nil +} + +// List returns known run IDs, most recently created first. +func (s *Store) List() ([]string, error) { + entries, err := os.ReadDir(s.baseDir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("checkpoint: list runs: %w", err) + } + type run struct { + id string + created time.Time + } + var runs []run + for _, e := range entries { + if !e.IsDir() { + continue + } + rs, err := s.Load(e.Name()) + if err != nil { + continue // not a run directory (or a corrupt one) - skip it + } + runs = append(runs, run{rs.RunID, rs.CreatedAt}) + } + sort.Slice(runs, func(i, j int) bool { return runs[i].created.After(runs[j].created) }) + ids := make([]string, len(runs)) + for i, r := range runs { + ids[i] = r.id + } + return ids, nil +} + +// RunStep executes fn for (phase, name) unless it already completed +// successfully in a prior attempt at this run, in which case fn is skipped +// and the previously recorded StepOutcome is returned instead - this is +// what makes an interrupted run resumable without redoing (or worse, +// double-applying) work that already happened. State is persisted both +// before fn runs (so a crash during fn is visible as "running", not silently +// forgotten) and after (recording success or failure). +func (s *Store) RunStep(rs *RunState, phase Phase, name string, fn func() (StepOutcome, error)) (StepOutcome, error) { + if rs.Done(phase, name) { + return rs.Outcome(phase, name), nil + } + rs.Begin(phase, name) + if err := s.Save(rs); err != nil { + return StepOutcome{}, fmt.Errorf("checkpoint: persist step start for %s/%s: %w", phase, name, err) + } + outcome, stepErr := fn() + if stepErr != nil { + rs.Fail(phase, name, stepErr) + } else { + rs.Complete(phase, name, outcome) + } + if err := s.Save(rs); err != nil { + if stepErr != nil { + return outcome, fmt.Errorf("%w (additionally failed to persist checkpoint: %v)", stepErr, err) + } + return outcome, fmt.Errorf("step %s/%s succeeded but failed to persist checkpoint: %w", phase, name, err) + } + return outcome, stepErr +} + +func newRunID() (string, error) { + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + return "", err + } + return fmt.Sprintf("%s-%s", time.Now().UTC().Format("20060102-150405"), hex.EncodeToString(b)), nil +} diff --git a/internal/checkpoint/store_test.go b/internal/checkpoint/store_test.go new file mode 100644 index 0000000..73131db --- /dev/null +++ b/internal/checkpoint/store_test.go @@ -0,0 +1,168 @@ +package checkpoint + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestCreateSaveLoadRoundtrip(t *testing.T) { + store := NewStore(t.TempDir()) + + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("Create: %v", err) + } + if rs.RunID == "" { + t.Fatal("Create: expected a non-empty run id") + } + + rs.Topology = Topology{DeploymentKind: "systemd", StoreBackend: "rocksdb"} + rs.RecordArtifact("fs-backup", Artifact{Path: "/var/lib/stalwart.v0155-backup", SHA256: "deadbeef", SizeBytes: 1024}) + if err := store.Save(rs); err != nil { + t.Fatalf("Save: %v", err) + } + + loaded, err := store.Load(rs.RunID) + if err != nil { + t.Fatalf("Load: %v", err) + } + if loaded.SourceVersion != "0.15.5" || loaded.TargetVersion != "0.16.14" { + t.Errorf("Load: versions = %s -> %s, want 0.15.5 -> 0.16.14", loaded.SourceVersion, loaded.TargetVersion) + } + if loaded.Topology.DeploymentKind != "systemd" { + t.Errorf("Load: DeploymentKind = %q, want systemd", loaded.Topology.DeploymentKind) + } + if got := loaded.Artifacts["fs-backup"].SHA256; got != "deadbeef" { + t.Errorf("Load: artifact sha256 = %q, want deadbeef", got) + } +} + +func TestSaveIsAtomicNoLeftoverTempFiles(t *testing.T) { + dir := t.TempDir() + store := NewStore(dir) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("Create: %v", err) + } + + for i := 0; i < 5; i++ { + rs.Begin(PhasePreflight, "version") + rs.Complete(PhasePreflight, "version", StepOutcome{Verdict: "ok", Detail: "ok"}) + if err := store.Save(rs); err != nil { + t.Fatalf("Save #%d: %v", i, err) + } + } + + entries, err := os.ReadDir(filepath.Join(dir, rs.RunID)) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + for _, e := range entries { + if e.Name() != "state.json" { + t.Errorf("unexpected leftover file in run dir: %s", e.Name()) + } + } +} + +func TestRunStepSkipsAlreadyDoneStep(t *testing.T) { + store := NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("Create: %v", err) + } + + calls := 0 + fn := func() (StepOutcome, error) { + calls++ + return StepOutcome{Verdict: "ok", Detail: "did the thing"}, nil + } + + outcome1, err := store.RunStep(rs, PhasePreflight, "version", fn) + if err != nil { + t.Fatalf("RunStep #1: %v", err) + } + if outcome1.Detail != "did the thing" { + t.Errorf("RunStep #1 detail = %q, want %q", outcome1.Detail, "did the thing") + } + if calls != 1 { + t.Fatalf("calls after first RunStep = %d, want 1", calls) + } + + // Simulate a resumed run: same rs, same step name. fn must not run again, + // and the previously recorded outcome must come back unchanged. + outcome2, err := store.RunStep(rs, PhasePreflight, "version", fn) + if err != nil { + t.Fatalf("RunStep #2 (resume): %v", err) + } + if calls != 1 { + t.Errorf("calls after resumed RunStep = %d, want 1 (fn should be skipped)", calls) + } + if outcome2 != outcome1 { + t.Errorf("RunStep #2 outcome = %+v, want %+v (unchanged from before)", outcome2, outcome1) + } +} + +func TestRunStepRetriesAfterFailure(t *testing.T) { + store := NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("Create: %v", err) + } + + calls := 0 + failThenSucceed := func() (StepOutcome, error) { + calls++ + if calls == 1 { + return StepOutcome{}, errors.New("transient failure") + } + return StepOutcome{Verdict: "ok", Detail: "succeeded on retry"}, nil + } + + if _, err := store.RunStep(rs, PhaseBackup, "fs-snapshot", failThenSucceed); err == nil { + t.Fatal("RunStep #1: expected error, got nil") + } + if rs.Status(PhaseBackup, "fs-snapshot") != StepFailed { + t.Errorf("status after failed attempt = %s, want failed", rs.Status(PhaseBackup, "fs-snapshot")) + } + + outcome, err := store.RunStep(rs, PhaseBackup, "fs-snapshot", failThenSucceed) + if err != nil { + t.Fatalf("RunStep #2 (retry): %v", err) + } + if calls != 2 { + t.Errorf("calls = %d, want 2 (failed step must retry, not skip)", calls) + } + if outcome.Detail != "succeeded on retry" { + t.Errorf("detail = %q, want %q", outcome.Detail, "succeeded on retry") + } + if !rs.Done(PhaseBackup, "fs-snapshot") { + t.Error("step should be Done after a successful retry") + } +} + +func TestListOrdersNewestFirst(t *testing.T) { + store := NewStore(t.TempDir()) + first, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("Create first: %v", err) + } + first.CreatedAt = first.CreatedAt.Add(-time.Hour) + if err := store.Save(first); err != nil { + t.Fatalf("Save first: %v", err) + } + second, err := store.Create("0.16.14", "0.16.15") + if err != nil { + t.Fatalf("Create second: %v", err) + } + + ids, err := store.List() + if err != nil { + t.Fatalf("List: %v", err) + } + if len(ids) != 2 || ids[0] != second.RunID || ids[1] != first.RunID { + t.Errorf("List = %v, want [%s %s]", ids, second.RunID, first.RunID) + } +} diff --git a/internal/checkpoint/types.go b/internal/checkpoint/types.go new file mode 100644 index 0000000..973bc49 --- /dev/null +++ b/internal/checkpoint/types.go @@ -0,0 +1,121 @@ +package checkpoint + +import "time" + +// Phase identifies one of the top-level migration phases from +// ARCHITECTURE.md §4. Step records are scoped to a phase so the same step +// name can be reused across phases without colliding. +type Phase string + +const ( + PhasePreflight Phase = "preflight" + PhaseBackup Phase = "backup" + PhaseStage Phase = "stage" + PhaseRecovery Phase = "recovery" + PhaseCutover Phase = "cutover" + PhaseValidate Phase = "validate" + PhaseRollback Phase = "rollback" +) + +// StepStatus is the lifecycle state of one checkpointed step. +type StepStatus string + +const ( + StepPending StepStatus = "pending" + StepRunning StepStatus = "running" + StepDone StepStatus = "done" + StepFailed StepStatus = "failed" +) + +// StepOutcome is what a step reports back on success: a verdict +// classification the calling phase defines the meaning of (e.g. preflight's +// "ok"/"warn"/"fail"), a human-readable summary, and an optional +// machine-readable value later steps - or a resumed run reconstructing this +// step's result without re-executing it - need. Keeping these three +// separate (rather than one free-text field) is what lets `stalwart-migrate +// status` print a clean human summary while still round-tripping the data a +// resumed run depends on. +type StepOutcome struct { + Verdict string `json:"verdict,omitempty"` + Detail string `json:"detail,omitempty"` + Extra string `json:"extra,omitempty"` +} + +// StepRecord captures one step's lifecycle status plus its StepOutcome. +type StepRecord struct { + Phase Phase `json:"phase"` + Name string `json:"name"` + Status StepStatus `json:"status"` + StepOutcome // embedded (untagged) so its fields flatten into this JSON object + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + Error string `json:"error,omitempty"` +} + +// Artifact is a content-addressed record of a file a run produced (a +// backup, a settings dump, a downloaded release binary) so later phases and +// a human operator can confirm it hasn't changed underfoot. +type Artifact struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + SizeBytes int64 `json:"size_bytes"` +} + +// MailboxCount is one mailbox's message count as observed at preflight +// time, for later comparison against the post-migration count. +type MailboxCount struct { + Mailbox string `json:"mailbox"` + Messages int `json:"messages"` +} + +// PreflightSnapshot holds the facts captured before anything is touched, +// which the validate phase later compares against the migrated instance. +// See ARCHITECTURE.md §4.1 and §4.7. +type PreflightSnapshot struct { + TakenAt time.Time `json:"taken_at"` + AccountCount int `json:"account_count"` + Domains []string `json:"domains,omitempty"` + MailboxCounts map[string][]MailboxCount `json:"mailbox_counts,omitempty"` // account -> mailboxes + DKIMFingerprints map[string]string `json:"dkim_fingerprints,omitempty"` + TLSFingerprints []string `json:"tls_fingerprints,omitempty"` + ListenerPorts []int `json:"listener_ports,omitempty"` +} + +// Topology records how this Stalwart instance is deployed, as detected +// during preflight, so later phases (cutover, rollback) know whether +// they're managing a systemd unit or a container and what backend they're +// dealing with. +type Topology struct { + DeploymentKind string `json:"deployment_kind,omitempty"` // "systemd", "docker", "unknown" + ClusterNodes []string `json:"cluster_nodes,omitempty"` + StoreBackend string `json:"store_backend,omitempty"` + BlobStore string `json:"blob_store,omitempty"` + FTSBackend string `json:"fts_backend,omitempty"` +} + +// RunState is the full persisted state of one migration run: everything +// needed to resume it after a crash, decide whether to roll back, or +// report on it later. See ARCHITECTURE.md §5. +type RunState struct { + RunID string `json:"run_id"` + SourceVersion string `json:"source_version"` + TargetVersion string `json:"target_version"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Topology Topology `json:"topology,omitempty"` + Steps []StepRecord `json:"steps"` + Artifacts map[string]Artifact `json:"artifacts,omitempty"` + PreflightSnapshot *PreflightSnapshot `json:"preflight_snapshot,omitempty"` + RollbackWindowClosed bool `json:"rollback_window_closed"` +} + +// RecordArtifact stores a content-addressed record of a file this run +// produced, keyed by a short logical name (e.g. "fs-backup", "settings-dump", +// "target-binary") rather than its path, since the path alone doesn't prove +// the content is what this run actually wrote. +func (rs *RunState) RecordArtifact(name string, a Artifact) { + if rs.Artifacts == nil { + rs.Artifacts = map[string]Artifact{} + } + rs.Artifacts[name] = a +} diff --git a/internal/config/doc.go b/internal/config/doc.go new file mode 100644 index 0000000..d8e254d --- /dev/null +++ b/internal/config/doc.go @@ -0,0 +1,3 @@ +// Package config implements the tool's own configuration: paths, thresholds, and credential handling. +// See ARCHITECTURE.md §6 for the design. +package config diff --git a/internal/plan/doc.go b/internal/plan/doc.go new file mode 100644 index 0000000..19afd5d --- /dev/null +++ b/internal/plan/doc.go @@ -0,0 +1,3 @@ +// Package plan implements the version-boundary migration plans (ordered step lists) that the engine executes. +// See ARCHITECTURE.md §4 for the design. +package plan diff --git a/internal/plan/plan.go b/internal/plan/plan.go new file mode 100644 index 0000000..bc84441 --- /dev/null +++ b/internal/plan/plan.go @@ -0,0 +1,122 @@ +package plan + +import ( + "fmt" + "regexp" + "strconv" +) + +// semver is a minimal major.minor.patch version - package-local like the +// equivalent in internal/preflight, since this comparison is the only thing +// plan needs from it and duplicating ~20 lines keeps phase packages +// independent per ARCHITECTURE.md §7. +type semver struct{ Major, Minor, Patch int } + +var versionPattern = regexp.MustCompile(`v?(\d+)\.(\d+)\.(\d+)`) + +func parseSemver(s string) (semver, error) { + m := versionPattern.FindStringSubmatch(s) + if m == nil { + return semver{}, fmt.Errorf("plan: no version number found in %q", s) + } + major, _ := strconv.Atoi(m[1]) + minor, _ := strconv.Atoi(m[2]) + patch, _ := strconv.Atoi(m[3]) + return semver{major, minor, patch}, nil +} + +func (v semver) String() string { return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) } + +func (v semver) Compare(o semver) int { + if v.Major != o.Major { + return cmp(v.Major, o.Major) + } + if v.Minor != o.Minor { + return cmp(v.Minor, o.Minor) + } + return cmp(v.Patch, o.Patch) +} + +func cmp(a, b int) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} + +// PhaseName identifies one phase in an ordered migration plan - the phase +// packages this names are internal/preflight, internal/backup, +// internal/recovery, internal/rollback (invoked on failure, not part of the +// forward list), and internal/validate. +type PhaseName string + +const ( + PhasePreflight PhaseName = "preflight" + PhaseBackup PhaseName = "backup" + PhaseRecovery PhaseName = "recovery" // only present when crossing the 0.15/0.16 boundary + PhaseCutover PhaseName = "cutover" + PhaseValidate PhaseName = "validate" +) + +// Plan is the ordered list of phases one migration run needs, decided once +// from the source and target versions. See ARCHITECTURE.md §4.6: crossing +// the 0.15/0.16 boundary needs the full recovery-mode migration (§4.4); a +// same-boundary patch bump (every 0.16.1-0.16.14 release so far, per +// Stalwart's own changelog) is the fast path with no recovery phase at all. +type Plan struct { + Phases []PhaseName + CrossesMajorBoundary bool + SourceVersion string + TargetVersion string + Reason string +} + +// HasPhase reports whether name appears in the plan. +func (p *Plan) HasPhase(name PhaseName) bool { + for _, ph := range p.Phases { + if ph == name { + return true + } + } + return false +} + +// Decide returns the plan for migrating from sourceVersion to +// targetVersion. It refuses (rather than guesses) if the source is already +// at or beyond the target, since that's not a migration this tool should +// attempt to run. +func Decide(sourceVersion, targetVersion string) (*Plan, error) { + src, err := parseSemver(sourceVersion) + if err != nil { + return nil, fmt.Errorf("plan: parse source version %q: %w", sourceVersion, err) + } + tgt, err := parseSemver(targetVersion) + if err != nil { + return nil, fmt.Errorf("plan: parse target version %q: %w", targetVersion, err) + } + if src.Compare(tgt) >= 0 { + return nil, fmt.Errorf("plan: source %s is already at or beyond target %s - nothing to migrate", src, tgt) + } + + crosses := src.Major == 0 && src.Minor < 16 && (tgt.Major > 0 || tgt.Minor >= 16) + if crosses { + return &Plan{ + Phases: []PhaseName{PhasePreflight, PhaseBackup, PhaseRecovery, PhaseCutover, PhaseValidate}, + CrossesMajorBoundary: true, + SourceVersion: src.String(), + TargetVersion: tgt.String(), + Reason: fmt.Sprintf("%s -> %s crosses the 0.15/0.16 major boundary: full recovery-mode migration required", src, tgt), + }, nil + } + return &Plan{ + Phases: []PhaseName{PhasePreflight, PhaseBackup, PhaseCutover, PhaseValidate}, + CrossesMajorBoundary: false, + SourceVersion: src.String(), + TargetVersion: tgt.String(), + Reason: fmt.Sprintf("%s -> %s is a same-boundary patch upgrade: fast path applies (no recovery-mode phase)", src, tgt), + }, nil +} diff --git a/internal/plan/plan_test.go b/internal/plan/plan_test.go new file mode 100644 index 0000000..79e6d2f --- /dev/null +++ b/internal/plan/plan_test.go @@ -0,0 +1,71 @@ +package plan + +import "testing" + +func TestDecideCrossesMajorBoundary(t *testing.T) { + p, err := Decide("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("Decide: %v", err) + } + if !p.CrossesMajorBoundary { + t.Error("CrossesMajorBoundary = false, want true for 0.15.5 -> 0.16.14") + } + if !p.HasPhase(PhaseRecovery) { + t.Errorf("phases %v missing PhaseRecovery", p.Phases) + } + wantOrder := []PhaseName{PhasePreflight, PhaseBackup, PhaseRecovery, PhaseCutover, PhaseValidate} + if len(p.Phases) != len(wantOrder) { + t.Fatalf("phases = %v, want %v", p.Phases, wantOrder) + } + for i, ph := range wantOrder { + if p.Phases[i] != ph { + t.Errorf("phases[%d] = %s, want %s", i, p.Phases[i], ph) + } + } +} + +func TestDecidePatchBumpFastPath(t *testing.T) { + p, err := Decide("0.16.5", "0.16.14") + if err != nil { + t.Fatalf("Decide: %v", err) + } + if p.CrossesMajorBoundary { + t.Error("CrossesMajorBoundary = true, want false for a 0.16.x -> 0.16.x bump") + } + if p.HasPhase(PhaseRecovery) { + t.Errorf("phases %v should not include PhaseRecovery on the fast path", p.Phases) + } +} + +func TestDecideRefusesNoOp(t *testing.T) { + if _, err := Decide("0.16.14", "0.16.14"); err == nil { + t.Fatal("Decide should refuse when source == target") + } +} + +func TestDecideRefusesRegression(t *testing.T) { + if _, err := Decide("0.16.14", "0.15.5"); err == nil { + t.Fatal("Decide should refuse when source is already beyond target") + } +} + +func TestDecideFutureMajorCrossesBoundaryToo(t *testing.T) { + // A hypothetical 0.15.x -> 1.0.0 jump should still be treated as + // crossing the boundary this tool knows how to automate. + p, err := Decide("0.15.9", "1.0.0") + if err != nil { + t.Fatalf("Decide: %v", err) + } + if !p.CrossesMajorBoundary { + t.Error("CrossesMajorBoundary = false, want true for 0.15.9 -> 1.0.0") + } +} + +func TestDecideRejectsUnparseableVersions(t *testing.T) { + if _, err := Decide("not-a-version", "0.16.14"); err == nil { + t.Fatal("Decide should reject an unparseable source version") + } + if _, err := Decide("0.15.5", "not-a-version"); err == nil { + t.Fatal("Decide should reject an unparseable target version") + } +} diff --git a/internal/preflight/checks.go b/internal/preflight/checks.go new file mode 100644 index 0000000..49eba04 --- /dev/null +++ b/internal/preflight/checks.go @@ -0,0 +1,290 @@ +package preflight + +import ( + "context" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" + "github.com/johnellis/stalwart-migrator/internal/stalwartapi" +) + +// Options configures a Checker. Every field has a conservative default +// applied by New except the ones that must name a real path on this host. +type Options struct { + BinaryPath string // installed stalwart binary, e.g. /usr/local/bin/stalwart + ConfigPath string // its config file (TOML pre-0.16, JSON 0.16+) + DataDir string // data directory to size/space-check + ContainerName string // docker container name, if applicable + AdminURL string // base URL for the JMAP reachability check; empty skips it + AdminUser string + AdminPassword string + TargetVersion string // e.g. "0.16.14" or "latest" + MinFreeMultiple float64 + HTTPClient *http.Client +} + +// Checker runs the preflight checks described in ARCHITECTURE.md §4.1. +type Checker struct { + opts Options +} + +func New(opts Options) *Checker { + if opts.MinFreeMultiple <= 0 { + opts.MinFreeMultiple = 2.0 + } + return &Checker{opts: opts} +} + +// Run executes every preflight check, checkpointing each one so a killed +// and re-invoked run skips checks that already completed - see +// checkpoint.Store.RunStep. It never aborts early on a single Fail: the +// point of preflight is to surface every blocking issue in one pass rather +// than fail-stop-fix-retry one at a time. Callers decide what to do with a +// Report whose Blocking() is true. It only returns a non-nil error for a +// genuine execution fault (e.g. the checkpoint store itself can't be +// written to) - a check finding a real problem is reported via +// Status: StatusFail in the Report, not a Go error. +func (c *Checker) Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState) (Report, error) { + var report Report + + // runCheck wraps fn as a checkpointed step and appends its result to + // report, whether fn actually ran or was skipped because a prior + // attempt already completed it - either way report ends up with the + // same entries, and the returned checkpoint.StepOutcome.Extra carries + // whatever machine-readable value a later check in this same Run needs. + runCheck := func(name string, fn func() (CheckResult, string)) (checkpoint.StepOutcome, error) { + outcome, err := store.RunStep(rs, checkpoint.PhasePreflight, name, func() (checkpoint.StepOutcome, error) { + res, extra := fn() + return checkpoint.StepOutcome{Verdict: string(res.Status), Detail: res.Detail, Extra: extra}, nil + }) + if err != nil { + return checkpoint.StepOutcome{}, err + } + report.Results = append(report.Results, CheckResult{Name: name, Status: Status(outcome.Verdict), Detail: outcome.Detail}) + return outcome, nil + } + + versionOutcome, err := runCheck("version", func() (CheckResult, string) { + cur, err := DetectVersion(ctx, c.opts.BinaryPath) + if err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + curV, _ := parseSemver(cur) + if curV.Compare(minSupportedSource) < 0 { + return CheckResult{ + Status: StatusFail, + Detail: fmt.Sprintf("current version %s is older than the minimum supported %s - upgrade to 0.15.x first", cur, minSupportedSource), + }, cur + } + return CheckResult{Status: StatusOK, Detail: fmt.Sprintf("current version %s", cur)}, cur + }) + if err != nil { + return report, err + } + + targetOutcome, err := runCheck("target-release", func() (CheckResult, string) { + rel, err := ResolveRelease(ctx, c.opts.HTTPClient, c.opts.TargetVersion) + if err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + tag := strings.TrimPrefix(rel.TagName, "v") + detail := fmt.Sprintf("resolved target to %s (%d release assets)", rel.TagName, len(rel.Assets)) + if asset := ChecksumAsset(rel); asset != nil { + detail += fmt.Sprintf(", checksum manifest available: %s", asset.Name) + } else { + detail += "; no published checksum manifest found - integrity relies on the one-time HTTPS download only" + } + return CheckResult{Status: StatusOK, Detail: detail}, tag + }) + if err != nil { + return report, err + } + + if _, err := runCheck("upgrade-direction", func() (CheckResult, string) { + curV, errCur := parseSemver(versionOutcome.Extra) + tgtV, errTgt := parseSemver(targetOutcome.Extra) + if errCur != nil || errTgt != nil { + return CheckResult{Status: StatusWarn, Detail: "could not compare current and target versions (one or both unresolved above)"}, "" + } + if curV.Compare(tgtV) >= 0 { + return CheckResult{ + Status: StatusFail, + Detail: fmt.Sprintf("current version %s is already at or beyond target %s - nothing to migrate", curV, tgtV), + }, "" + } + if curV.Major == 0 && curV.Minor < 16 && (tgtV.Major > 0 || tgtV.Minor >= 16) { + return CheckResult{ + Status: StatusOK, + Detail: fmt.Sprintf("%s -> %s crosses the 0.15/0.16 major boundary: full recovery-mode migration plan required (ARCHITECTURE.md §4.4)", curV, tgtV), + }, "" + } + return CheckResult{ + Status: StatusOK, + Detail: fmt.Sprintf("%s -> %s is a same-boundary patch upgrade: fast-path plan applies (ARCHITECTURE.md §4.6)", curV, tgtV), + }, "" + }); err != nil { + return report, err + } + + deploymentOutcome, err := runCheck("deployment-kind", func() (CheckResult, string) { + kind := DetectDeploymentKind(ctx, c.opts.ContainerName) + status := StatusOK + if kind == DeploymentUnknown { + status = StatusWarn + } + return CheckResult{Status: status, Detail: fmt.Sprintf("detected deployment kind: %s", kind)}, string(kind) + }) + if err != nil { + return report, err + } + + storeOutcome, err := runCheck("store-backend", func() (CheckResult, string) { + matches, err := DetectStoreBackends(c.opts.ConfigPath) + if err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + if len(matches) == 0 { + return CheckResult{Status: StatusWarn, Detail: "no known store backend type found in config - confirm manually before proceeding"}, "" + } + names := make([]string, len(matches)) + backends := make([]string, len(matches)) + for i, m := range matches { + names[i] = fmt.Sprintf("%s (%s)", m.Backend, m.Path) + backends[i] = m.Backend + } + return CheckResult{Status: StatusOK, Detail: "found: " + strings.Join(names, ", ")}, strings.Join(backends, ",") + }) + if err != nil { + return report, err + } + + if _, err := runCheck("cluster-config", func() (CheckResult, string) { + clustered, err := LooksClustered(c.opts.ConfigPath) + if err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + if clustered { + return CheckResult{ + Status: StatusWarn, + Detail: "config mentions clustering - confirm every peer node is stopped before this run proceeds; the tool does not verify this for you", + }, "" + } + return CheckResult{Status: StatusOK, Detail: "no cluster configuration detected"}, "" + }); err != nil { + return report, err + } + + if _, err := runCheck("disk-space", func() (CheckResult, string) { + size, err := DirSize(c.opts.DataDir) + if err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + free, err := FreeBytes(c.opts.DataDir) + if err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + required := uint64(float64(size) * c.opts.MinFreeMultiple) + detail := fmt.Sprintf("data dir %s is %s, %s free, need >= %s (%.1fx, for the fs-snapshot backup)", + c.opts.DataDir, humanBytes(uint64(size)), humanBytes(free), humanBytes(required), c.opts.MinFreeMultiple) + if free < required { + return CheckResult{Status: StatusFail, Detail: detail}, "" + } + return CheckResult{Status: StatusOK, Detail: detail}, "" + }); err != nil { + return report, err + } + + if c.opts.AdminURL != "" { + if _, err := runCheck("admin-reachable", func() (CheckResult, string) { + client := &stalwartapi.Client{ + BaseURL: c.opts.AdminURL, + Username: c.opts.AdminUser, + Password: c.opts.AdminPassword, + HTTPClient: c.opts.HTTPClient, + } + if err := client.Ping(ctx); err != nil { + return CheckResult{Status: StatusFail, Detail: err.Error()}, "" + } + return CheckResult{Status: StatusOK, Detail: fmt.Sprintf("JMAP session reachable at %s with the given credentials", c.opts.AdminURL)}, "" + }); err != nil { + return report, err + } + + if _, err := runCheck("account-snapshot", func() (CheckResult, string) { + client := &stalwartapi.Client{ + BaseURL: c.opts.AdminURL, + Username: c.opts.AdminUser, + Password: c.opts.AdminPassword, + HTTPClient: c.opts.HTTPClient, + } + snap, err := client.AccountSnapshot(ctx) + if err != nil { + return CheckResult{ + Status: StatusWarn, + Detail: fmt.Sprintf("could not capture the account/domain snapshot: %v - the post-migration directory-integrity check won't have anything to compare against", err), + }, "" + } + mailboxCounts := make(map[string][]checkpoint.MailboxCount, len(snap.MailboxCounts)) + for account, counts := range snap.MailboxCounts { + converted := make([]checkpoint.MailboxCount, len(counts)) + for i, mc := range counts { + converted[i] = checkpoint.MailboxCount{Mailbox: mc.Mailbox, Messages: mc.Messages} + } + mailboxCounts[account] = converted + } + rs.PreflightSnapshot = &checkpoint.PreflightSnapshot{ + TakenAt: time.Now().UTC(), + AccountCount: snap.AccountCount, + Domains: snap.Domains, + MailboxCounts: mailboxCounts, + } + detail := fmt.Sprintf("captured snapshot: %d account(s) across %d domain(s), mailbox counts for %d account(s)", + snap.AccountCount, len(snap.Domains), len(mailboxCounts)) + status := StatusOK + if len(snap.MailboxErrors) > 0 { + status = StatusWarn + accounts := make([]string, 0, len(snap.MailboxErrors)) + for account := range snap.MailboxErrors { + accounts = append(accounts, account) + } + sort.Strings(accounts) + for i, account := range accounts { + if i >= 3 { + detail += fmt.Sprintf(" (and %d more)", len(accounts)-3) + break + } + detail += fmt.Sprintf("; mailbox count failed for %s: %s", account, snap.MailboxErrors[account]) + } + } + return CheckResult{Status: status, Detail: detail}, "" + }); err != nil { + return report, err + } + } else { + report.Results = append(report.Results, CheckResult{ + Name: "admin-reachable", + Status: StatusWarn, + Detail: "no --admin-url configured - skipped; the account/mailbox snapshot validate needs later can't be captured without it", + }) + } + + rs.Topology = checkpoint.Topology{ + DeploymentKind: deploymentOutcome.Extra, + StoreBackend: storeOutcome.Extra, + } + if versionOutcome.Extra != "" { + rs.SourceVersion = versionOutcome.Extra + } + if targetOutcome.Extra != "" { + rs.TargetVersion = targetOutcome.Extra + } + if err := store.Save(rs); err != nil { + return report, fmt.Errorf("preflight: persist topology: %w", err) + } + + return report, nil +} diff --git a/internal/preflight/checks_test.go b/internal/preflight/checks_test.go new file mode 100644 index 0000000..4445555 --- /dev/null +++ b/internal/preflight/checks_test.go @@ -0,0 +1,304 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" +) + +// writeFakeBinary creates a shell script that behaves like `stalwart +// --version` and records each invocation to counterPath, so tests can +// assert a checkpointed step was (or wasn't) re-executed on resume. +func writeFakeBinary(t *testing.T, version, counterPath string) string { + t.Helper() + dir := t.TempDir() + scriptPath := filepath.Join(dir, "fake-stalwart.sh") + script := fmt.Sprintf("#!/bin/sh\necho invoked >> %q\necho 'stalwart %s'\n", counterPath, version) + if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return scriptPath +} + +func countLines(t *testing.T, path string) int { + t.Helper() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return 0 + } + if err != nil { + t.Fatal(err) + } + return len(strings.Split(strings.TrimSpace(string(data)), "\n")) +} + +func TestCheckerRunEndToEndAndResume(t *testing.T) { + // -- fixtures -------------------------------------------------------- + counterPath := filepath.Join(t.TempDir(), "invocations") + binaryPath := writeFakeBinary(t, "0.15.5", counterPath) + + configPath := filepath.Join(t.TempDir(), "config.toml") + tomlCfg := "[store.\"rocksdb\"]\ntype = \"rocksdb\"\npath = \"/var/lib/stalwart/data\"\n" + if err := os.WriteFile(configPath, []byte(tomlCfg), 0o644); err != nil { + t.Fatal(err) + } + + dataDir := t.TempDir() + if err := os.WriteFile(filepath.Join(dataDir, "db"), make([]byte, 1024), 0o644); err != nil { + t.Fatal(err) + } + + withFakeGithub(t, func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(Release{ + TagName: "v0.16.14", + Assets: []ReleaseAsset{{Name: "checksums.txt"}}, + }) + }) + + stateDir := t.TempDir() + store := checkpoint.NewStore(stateDir) + rs, err := store.Create("", "latest") + if err != nil { + t.Fatalf("store.Create: %v", err) + } + + checker := New(Options{ + BinaryPath: binaryPath, + ConfigPath: configPath, + DataDir: dataDir, + TargetVersion: "latest", + }) + + // -- first run: everything should execute ----------------------------- + report, err := checker.Run(context.Background(), store, rs) + if err != nil { + t.Fatalf("Run #1: %v", err) + } + if report.Blocking() { + t.Fatalf("Run #1: unexpected blocking report:\n%s", report.String()) + } + if got := countLines(t, counterPath); got != 1 { + t.Fatalf("binary invocations after Run #1 = %d, want 1", got) + } + if rs.SourceVersion != "0.15.5" { + t.Errorf("rs.SourceVersion = %q, want 0.15.5", rs.SourceVersion) + } + if rs.TargetVersion != "0.16.14" { + t.Errorf("rs.TargetVersion = %q, want 0.16.14 (resolved from \"latest\")", rs.TargetVersion) + } + if rs.Topology.StoreBackend != "rocksdb" { + t.Errorf("rs.Topology.StoreBackend = %q, want rocksdb", rs.Topology.StoreBackend) + } + + foundDirection := false + for _, res := range report.Results { + if res.Name == "upgrade-direction" { + foundDirection = true + if !strings.Contains(res.Detail, "major boundary") { + t.Errorf("upgrade-direction detail = %q, want mention of the major boundary", res.Detail) + } + } + } + if !foundDirection { + t.Error("report missing upgrade-direction check") + } + + // -- simulate a crash and resume: reload state from disk fresh -------- + resumed, err := store.Load(rs.RunID) + if err != nil { + t.Fatalf("store.Load (resume): %v", err) + } + report2, err := checker.Run(context.Background(), store, resumed) + if err != nil { + t.Fatalf("Run #2 (resume): %v", err) + } + if got := countLines(t, counterPath); got != 1 { + t.Errorf("binary invocations after resumed Run = %d, want 1 (already-done steps must not re-execute)", got) + } + if len(report2.Results) != len(report.Results) { + t.Errorf("resumed report has %d results, want %d (same as first run)", len(report2.Results), len(report.Results)) + } +} + +func TestCheckerRunFlagsTooOldSource(t *testing.T) { + counterPath := filepath.Join(t.TempDir(), "invocations") + binaryPath := writeFakeBinary(t, "0.14.2", counterPath) + + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte("[server]\nhostname = \"mail.example.com\"\n"), 0o644); err != nil { + t.Fatal(err) + } + dataDir := t.TempDir() + + withFakeGithub(t, func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(Release{TagName: "v0.16.14"}) + }) + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("", "latest") + if err != nil { + t.Fatal(err) + } + checker := New(Options{BinaryPath: binaryPath, ConfigPath: configPath, DataDir: dataDir, TargetVersion: "latest"}) + + report, err := checker.Run(context.Background(), store, rs) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !report.Blocking() { + t.Fatalf("expected a blocking report for a too-old source version, got:\n%s", report.String()) + } +} + +func TestCheckerRunFlagsInsufficientDiskSpace(t *testing.T) { + counterPath := filepath.Join(t.TempDir(), "invocations") + binaryPath := writeFakeBinary(t, "0.15.5", counterPath) + + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte("[store.\"rocksdb\"]\ntype = \"rocksdb\"\n"), 0o644); err != nil { + t.Fatal(err) + } + dataDir := t.TempDir() + if err := os.WriteFile(filepath.Join(dataDir, "db"), make([]byte, 1024), 0o644); err != nil { + t.Fatal(err) + } + + withFakeGithub(t, func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(Release{TagName: "v0.16.14"}) + }) + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("", "latest") + if err != nil { + t.Fatal(err) + } + // An absurd multiple guarantees the free-space check fails regardless + // of how much space the test host actually has free. + checker := New(Options{ + BinaryPath: binaryPath, ConfigPath: configPath, DataDir: dataDir, + TargetVersion: "latest", MinFreeMultiple: 1e12, + }) + + report, err := checker.Run(context.Background(), store, rs) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !report.Blocking() { + t.Fatalf("expected a blocking report for insufficient disk space, got:\n%s", report.String()) + } +} + +func TestCheckerRunCapturesAccountSnapshotWhenAdminURLSet(t *testing.T) { + counterPath := filepath.Join(t.TempDir(), "invocations") + binaryPath := writeFakeBinary(t, "0.15.5", counterPath) + + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte("[store.\"rocksdb\"]\ntype = \"rocksdb\"\n"), 0o644); err != nil { + t.Fatal(err) + } + dataDir := t.TempDir() + if err := os.WriteFile(filepath.Join(dataDir, "db"), make([]byte, 1024), 0o644); err != nil { + t.Fatal(err) + } + + withFakeGithub(t, func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(Release{TagName: "v0.16.14"}) + }) + + // A fake admin server: answers Ping's session-discovery GET, the + // impersonated session-discovery GET MailboxSnapshot makes for + // alice@example.com, and the POST /api calls for x:Account/query, + // x:Account/get, and Mailbox/get. + var apiURL string + adminSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap": + user, _, _ := r.BasicAuth() + if strings.Contains(user, "%") { + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "primaryAccounts": map[string]string{"urn:ietf:params:jmap:mail": "mail-alice"}, + }) + return + } + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api": + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + methodCalls := body["methodCalls"].([]any) + name := methodCalls[0].([]any)[0].(string) + switch name { + case "x:Account/query": + json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{ + []any{"x:Account/query", map[string]any{"ids": []string{"a1"}}, "q"}, + }}) + case "x:Account/get": + json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{ + []any{"x:Account/get", map[string]any{"list": []map[string]any{ + {"id": "a1", "name": "alice@example.com", "domainId": "example.com"}, + }}, "g"}, + }}) + case "Mailbox/get": + json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{ + []any{"Mailbox/get", map[string]any{"list": []map[string]any{ + {"name": "Inbox", "totalEmails": 5}, + }}, "m"}, + }}) + } + default: + w.WriteHeader(http.StatusNotFound) + } + })) + apiURL = adminSrv.URL + "/api" + defer adminSrv.Close() + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("", "latest") + if err != nil { + t.Fatal(err) + } + checker := New(Options{ + BinaryPath: binaryPath, ConfigPath: configPath, DataDir: dataDir, TargetVersion: "latest", + AdminURL: adminSrv.URL, AdminUser: "admin", AdminPassword: "hunter2", + }) + + report, err := checker.Run(context.Background(), store, rs) + if err != nil { + t.Fatalf("Run: %v", err) + } + if report.Blocking() { + t.Fatalf("unexpected blocking report:\n%s", report.String()) + } + if rs.PreflightSnapshot == nil { + t.Fatal("PreflightSnapshot should be populated when AdminURL is set") + } + if rs.PreflightSnapshot.AccountCount != 1 { + t.Errorf("AccountCount = %d, want 1", rs.PreflightSnapshot.AccountCount) + } + if len(rs.PreflightSnapshot.Domains) != 1 || rs.PreflightSnapshot.Domains[0] != "example.com" { + t.Errorf("Domains = %v, want [example.com]", rs.PreflightSnapshot.Domains) + } + aliceMailboxes := rs.PreflightSnapshot.MailboxCounts["alice@example.com"] + if len(aliceMailboxes) != 1 || aliceMailboxes[0].Mailbox != "Inbox" || aliceMailboxes[0].Messages != 5 { + t.Errorf("alice's mailbox counts = %+v, want [{Inbox 5}]", aliceMailboxes) + } + + // Resume: the snapshot must survive a reload from disk without + // re-running the check (the admin server would still work, but this + // confirms the persisted value is what's actually being relied on). + resumed, err := store.Load(rs.RunID) + if err != nil { + t.Fatalf("store.Load: %v", err) + } + if resumed.PreflightSnapshot == nil || resumed.PreflightSnapshot.AccountCount != 1 { + t.Errorf("resumed PreflightSnapshot = %+v, want AccountCount 1", resumed.PreflightSnapshot) + } +} diff --git a/internal/preflight/cluster.go b/internal/preflight/cluster.go new file mode 100644 index 0000000..7e86245 --- /dev/null +++ b/internal/preflight/cluster.go @@ -0,0 +1,22 @@ +package preflight + +import ( + "fmt" + "os" + "strings" +) + +// LooksClustered does a conservative, heuristic scan for cluster-related +// configuration. It exists to force a manual confirmation gate (see +// ARCHITECTURE.md §4.1's cluster gate: one live node on the old version +// during migration corrupts a shared store), not to enumerate peers +// precisely. A false positive just costs an extra confirmation prompt; a +// false negative is the dangerous direction, so this errs toward matching +// broadly rather than requiring an exact schema match. +func LooksClustered(configPath string) (bool, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return false, fmt.Errorf("preflight: read config %s: %w", configPath, err) + } + return strings.Contains(strings.ToLower(string(data)), "cluster"), nil +} diff --git a/internal/preflight/cluster_test.go b/internal/preflight/cluster_test.go new file mode 100644 index 0000000..4ad6cef --- /dev/null +++ b/internal/preflight/cluster_test.go @@ -0,0 +1,34 @@ +package preflight + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLooksClustered(t *testing.T) { + cases := []struct { + name string + content string + want bool + }{ + {"no-cluster", "[server]\nhostname = \"mail.example.com\"\n", false}, + {"has-cluster-section", "[cluster]\nnode-id = 1\npeers = [\"10.0.0.2\"]\n", true}, + {"cluster-mentioned-in-key", "[server]\ncluster-coordinator = \"redis://localhost\"\n", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { + t.Fatal(err) + } + got, err := LooksClustered(path) + if err != nil { + t.Fatalf("LooksClustered: %v", err) + } + if got != tc.want { + t.Errorf("LooksClustered(%s) = %v, want %v", tc.name, got, tc.want) + } + }) + } +} diff --git a/internal/preflight/deployment.go b/internal/preflight/deployment.go new file mode 100644 index 0000000..867e9a9 --- /dev/null +++ b/internal/preflight/deployment.go @@ -0,0 +1,45 @@ +package preflight + +import ( + "context" + "os" + "os/exec" +) + +// DeploymentKind is how a Stalwart instance appears to be run, which +// determines how cutover and rollback restart it. +type DeploymentKind string + +const ( + DeploymentSystemd DeploymentKind = "systemd" + DeploymentDocker DeploymentKind = "docker" + DeploymentUnknown DeploymentKind = "unknown" +) + +var systemdUnitPaths = []string{ + "/etc/systemd/system/stalwart.service", + "/lib/systemd/system/stalwart.service", + "/usr/lib/systemd/system/stalwart.service", +} + +// DetectDeploymentKind makes a best-effort guess at how Stalwart is run +// here. It's deliberately conservative and cheap (file stats, one docker +// inspect) rather than exhaustive - an operator-supplied override should +// always be able to win over this, since the cost of guessing wrong here is +// cutover targeting the wrong thing. +func DetectDeploymentKind(ctx context.Context, containerName string) DeploymentKind { + for _, p := range systemdUnitPaths { + if _, err := os.Stat(p); err == nil { + return DeploymentSystemd + } + } + if containerName == "" { + containerName = "stalwart" + } + if _, err := exec.LookPath("docker"); err == nil { + if err := exec.CommandContext(ctx, "docker", "inspect", containerName).Run(); err == nil { + return DeploymentDocker + } + } + return DeploymentUnknown +} diff --git a/internal/preflight/diskspace.go b/internal/preflight/diskspace.go new file mode 100644 index 0000000..26160bc --- /dev/null +++ b/internal/preflight/diskspace.go @@ -0,0 +1,55 @@ +package preflight + +import ( + "fmt" + "io/fs" + "path/filepath" + "syscall" +) + +// DirSize walks dir and sums the size of every regular file in it. Used to +// estimate how much free space a filesystem-level backup copy will need - +// see ARCHITECTURE.md §4.2. +func DirSize(dir string) (int64, error) { + var total int64 + err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + info, err := d.Info() + if err != nil { + return err + } + total += info.Size() + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("preflight: measure size of %s: %w", dir, err) + } + return total, nil +} + +// FreeBytes returns the free space available (to an unprivileged process) +// on the filesystem containing path. +func FreeBytes(path string) (uint64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, fmt.Errorf("preflight: statfs %s: %w", path, err) + } + return stat.Bavail * uint64(stat.Bsize), nil +} + +func humanBytes(n uint64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := uint64(unit), 0 + for n/div >= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/internal/preflight/diskspace_test.go b/internal/preflight/diskspace_test.go new file mode 100644 index 0000000..5fb0b09 --- /dev/null +++ b/internal/preflight/diskspace_test.go @@ -0,0 +1,55 @@ +package preflight + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDirSize(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.bin"), make([]byte, 100), 0o644); err != nil { + t.Fatal(err) + } + sub := filepath.Join(dir, "sub") + if err := os.Mkdir(sub, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sub, "b.bin"), make([]byte, 250), 0o644); err != nil { + t.Fatal(err) + } + + got, err := DirSize(dir) + if err != nil { + t.Fatalf("DirSize: %v", err) + } + if got != 350 { + t.Errorf("DirSize = %d, want 350", got) + } +} + +func TestFreeBytes(t *testing.T) { + got, err := FreeBytes(t.TempDir()) + if err != nil { + t.Fatalf("FreeBytes: %v", err) + } + if got == 0 { + t.Error("FreeBytes = 0, want > 0 for a live filesystem") + } +} + +func TestHumanBytes(t *testing.T) { + cases := []struct { + in uint64 + want string + }{ + {500, "500 B"}, + {1536, "1.5 KiB"}, + {5 * 1024 * 1024, "5.0 MiB"}, + } + for _, tc := range cases { + if got := humanBytes(tc.in); got != tc.want { + t.Errorf("humanBytes(%d) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/preflight/doc.go b/internal/preflight/doc.go new file mode 100644 index 0000000..1c93fb0 --- /dev/null +++ b/internal/preflight/doc.go @@ -0,0 +1,3 @@ +// Package preflight implements the read-only preflight checks that gate a run before anything is touched. +// See ARCHITECTURE.md §4.1 for the design. +package preflight diff --git a/internal/preflight/release.go b/internal/preflight/release.go new file mode 100644 index 0000000..69212b3 --- /dev/null +++ b/internal/preflight/release.go @@ -0,0 +1,74 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// githubAPIBase is a var, not a const, so tests can point it at an +// httptest server instead of hitting the real GitHub API. +var githubAPIBase = "https://api.github.com/repos/stalwartlabs/stalwart/releases" + +type ReleaseAsset struct { + Name string `json:"name"` + DownloadURL string `json:"browser_download_url"` + SizeBytes int64 `json:"size"` +} + +type Release struct { + TagName string `json:"tag_name"` + Assets []ReleaseAsset `json:"assets"` +} + +// ResolveRelease looks up a Stalwart release from the public GitHub API. +// version is either an exact tag like "0.16.14" (a "v" prefix is added if +// missing) or "latest". +func ResolveRelease(ctx context.Context, httpClient *http.Client, version string) (*Release, error) { + url := githubAPIBase + "/latest" + if version != "" && version != "latest" { + tag := version + if !strings.HasPrefix(tag, "v") { + tag = "v" + tag + } + url = githubAPIBase + "/tags/" + tag + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + if httpClient == nil { + httpClient = &http.Client{Timeout: 30 * time.Second} + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("preflight: fetch release %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("preflight: fetch release %s: unexpected status %s", url, resp.Status) + } + var rel Release + if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil { + return nil, fmt.Errorf("preflight: parse release response from %s: %w", url, err) + } + return &rel, nil +} + +// ChecksumAsset returns the release asset that looks like a published +// checksum manifest, if any. Its presence isn't guaranteed by Stalwart's +// release process, so callers must treat a nil result as "no independently +// published checksum to verify the download against", not an error. +func ChecksumAsset(rel *Release) *ReleaseAsset { + for i := range rel.Assets { + name := strings.ToLower(rel.Assets[i].Name) + if strings.Contains(name, "sha256") || strings.Contains(name, "checksum") { + return &rel.Assets[i] + } + } + return nil +} diff --git a/internal/preflight/release_test.go b/internal/preflight/release_test.go new file mode 100644 index 0000000..f0b5519 --- /dev/null +++ b/internal/preflight/release_test.go @@ -0,0 +1,68 @@ +package preflight + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func withFakeGithub(t *testing.T, handler http.HandlerFunc) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + old := githubAPIBase + githubAPIBase = srv.URL + t.Cleanup(func() { githubAPIBase = old }) +} + +func TestResolveReleaseLatest(t *testing.T) { + withFakeGithub(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/latest" { + t.Errorf("path = %s, want /latest", r.URL.Path) + } + json.NewEncoder(w).Encode(Release{ + TagName: "v0.16.14", + Assets: []ReleaseAsset{ + {Name: "stalwart-x86_64-linux", DownloadURL: "https://example.com/stalwart"}, + {Name: "checksums.txt", DownloadURL: "https://example.com/checksums.txt"}, + }, + }) + }) + + rel, err := ResolveRelease(context.Background(), nil, "latest") + if err != nil { + t.Fatalf("ResolveRelease: %v", err) + } + if rel.TagName != "v0.16.14" { + t.Errorf("TagName = %s, want v0.16.14", rel.TagName) + } + if asset := ChecksumAsset(rel); asset == nil || asset.Name != "checksums.txt" { + t.Errorf("ChecksumAsset = %+v, want checksums.txt", asset) + } +} + +func TestResolveReleaseExactTag(t *testing.T) { + withFakeGithub(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/tags/v0.16.5" { + t.Errorf("path = %s, want /tags/v0.16.5", r.URL.Path) + } + json.NewEncoder(w).Encode(Release{TagName: "v0.16.5"}) + }) + + rel, err := ResolveRelease(context.Background(), nil, "0.16.5") + if err != nil { + t.Fatalf("ResolveRelease: %v", err) + } + if rel.TagName != "v0.16.5" { + t.Errorf("TagName = %s, want v0.16.5", rel.TagName) + } +} + +func TestChecksumAssetAbsent(t *testing.T) { + rel := &Release{Assets: []ReleaseAsset{{Name: "stalwart-x86_64-linux"}}} + if asset := ChecksumAsset(rel); asset != nil { + t.Errorf("ChecksumAsset = %+v, want nil", asset) + } +} diff --git a/internal/preflight/report.go b/internal/preflight/report.go new file mode 100644 index 0000000..fac526b --- /dev/null +++ b/internal/preflight/report.go @@ -0,0 +1,46 @@ +package preflight + +import ( + "fmt" + "strings" +) + +// Status is a single check's verdict. +type Status string + +const ( + StatusOK Status = "ok" + StatusWarn Status = "warn" + StatusFail Status = "fail" +) + +// CheckResult is one named check's outcome. +type CheckResult struct { + Name string + Status Status + Detail string +} + +// Report is the full set of preflight check outcomes. +type Report struct { + Results []CheckResult +} + +// Blocking reports whether any check failed hard enough that a run must +// not proceed. +func (r Report) Blocking() bool { + for _, res := range r.Results { + if res.Status == StatusFail { + return true + } + } + return false +} + +func (r Report) String() string { + var b strings.Builder + for _, res := range r.Results { + fmt.Fprintf(&b, "[%-4s] %-20s %s\n", strings.ToUpper(string(res.Status)), res.Name, res.Detail) + } + return b.String() +} diff --git a/internal/preflight/storebackend.go b/internal/preflight/storebackend.go new file mode 100644 index 0000000..08e935f --- /dev/null +++ b/internal/preflight/storebackend.go @@ -0,0 +1,106 @@ +package preflight + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "os" + "regexp" + "strings" +) + +// knownBackends is the set of store/blob/FTS backend identifiers Stalwart's +// documentation names: RocksDB, SQLite, FoundationDB, PostgreSQL, MySQL, +// S3-compatible storage, and Elasticsearch. +var knownBackends = map[string]bool{ + "rocksdb": true, "sqlite": true, "foundationdb": true, + "postgresql": true, "mysql": true, "s3": true, "elasticsearch": true, +} + +// BackendMatch is one "type = " assignment found in a config +// file, tagged with where it was found (the enclosing TOML section, or the +// dotted JSON key path). +type BackendMatch struct { + Path string + Backend string +} + +// DetectStoreBackends scans a Stalwart config file for store/blob/FTS +// backend declarations. It deliberately does not assume one fixed schema +// path: the exact TOML (pre-0.16) or JSON (0.16+) layout has already +// changed once between those versions and may change again, so this +// searches structurally for `type = ""` assignments wherever +// they appear and reports every match with its location. Treat the result +// as "here's what to confirm before an unattended run", not ground truth. +func DetectStoreBackends(configPath string) ([]BackendMatch, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("preflight: read config %s: %w", configPath, err) + } + if json.Valid(data) { + return scanJSONBackends(data) + } + return scanTOMLBackends(data) +} + +var ( + tomlSectionRe = regexp.MustCompile(`^\[(.+)\]$`) + tomlKVRe = regexp.MustCompile(`^([A-Za-z0-9_.-]+)\s*=\s*"([^"]*)"$`) +) + +func scanTOMLBackends(data []byte) ([]BackendMatch, error) { + var matches []BackendMatch + section := "" + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if m := tomlSectionRe.FindStringSubmatch(line); m != nil { + section = m[1] + continue + } + if m := tomlKVRe.FindStringSubmatch(line); m != nil { + key, value := m[1], strings.ToLower(m[2]) + if key == "type" && knownBackends[value] { + matches = append(matches, BackendMatch{Path: section, Backend: value}) + } + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("preflight: scan toml config: %w", err) + } + return matches, nil +} + +func scanJSONBackends(data []byte) ([]BackendMatch, error) { + var root any + if err := json.Unmarshal(data, &root); err != nil { + return nil, fmt.Errorf("preflight: parse json config: %w", err) + } + var matches []BackendMatch + walkJSONForBackends(root, "", &matches) + return matches, nil +} + +func walkJSONForBackends(node any, path string, matches *[]BackendMatch) { + switch v := node.(type) { + case map[string]any: + if t, ok := v["type"].(string); ok && knownBackends[strings.ToLower(t)] { + *matches = append(*matches, BackendMatch{Path: path, Backend: strings.ToLower(t)}) + } + for k, child := range v { + childPath := k + if path != "" { + childPath = path + "." + k + } + walkJSONForBackends(child, childPath, matches) + } + case []any: + for i, child := range v { + walkJSONForBackends(child, fmt.Sprintf("%s[%d]", path, i), matches) + } + } +} diff --git a/internal/preflight/storebackend_test.go b/internal/preflight/storebackend_test.go new file mode 100644 index 0000000..59f2d53 --- /dev/null +++ b/internal/preflight/storebackend_test.go @@ -0,0 +1,86 @@ +package preflight + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDetectStoreBackendsTOML(t *testing.T) { + toml := ` +[server] +hostname = "mail.example.com" + +[store."rocksdb"] +type = "rocksdb" +path = "/var/lib/stalwart/data" + +[store."blob"] +type = "s3" +bucket = "stalwart-blobs" +` + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(toml), 0o644); err != nil { + t.Fatal(err) + } + + matches, err := DetectStoreBackends(path) + if err != nil { + t.Fatalf("DetectStoreBackends: %v", err) + } + if len(matches) != 2 { + t.Fatalf("got %d matches, want 2: %+v", len(matches), matches) + } + got := map[string]string{} + for _, m := range matches { + got[m.Path] = m.Backend + } + if got[`store."rocksdb"`] != "rocksdb" { + t.Errorf(`store."rocksdb" backend = %q, want rocksdb`, got[`store."rocksdb"`]) + } + if got[`store."blob"`] != "s3" { + t.Errorf(`store."blob" backend = %q, want s3`, got[`store."blob"`]) + } +} + +func TestDetectStoreBackendsJSON(t *testing.T) { + jsonCfg := `{ + "store": { + "data": {"type": "postgresql", "host": "db.internal"}, + "blob": {"type": "s3", "bucket": "stalwart-blobs"} + } + }` + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(jsonCfg), 0o644); err != nil { + t.Fatal(err) + } + + matches, err := DetectStoreBackends(path) + if err != nil { + t.Fatalf("DetectStoreBackends: %v", err) + } + if len(matches) != 2 { + t.Fatalf("got %d matches, want 2: %+v", len(matches), matches) + } + backends := map[string]bool{} + for _, m := range matches { + backends[m.Backend] = true + } + if !backends["postgresql"] || !backends["s3"] { + t.Errorf("matches = %+v, want postgresql and s3", matches) + } +} + +func TestDetectStoreBackendsNoMatch(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte("[server]\nhostname = \"mail.example.com\"\n"), 0o644); err != nil { + t.Fatal(err) + } + matches, err := DetectStoreBackends(path) + if err != nil { + t.Fatalf("DetectStoreBackends: %v", err) + } + if len(matches) != 0 { + t.Errorf("got %d matches, want 0: %+v", len(matches), matches) + } +} diff --git a/internal/preflight/version.go b/internal/preflight/version.go new file mode 100644 index 0000000..5058664 --- /dev/null +++ b/internal/preflight/version.go @@ -0,0 +1,78 @@ +package preflight + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// semver is a minimal major.minor.patch version - enough for this tool's +// comparisons since Stalwart's pre-1.0 release tags don't use pre-release +// or build metadata. +type semver struct { + Major, Minor, Patch int +} + +var versionPattern = regexp.MustCompile(`v?(\d+)\.(\d+)\.(\d+)`) + +func parseSemver(s string) (semver, error) { + m := versionPattern.FindStringSubmatch(s) + if m == nil { + return semver{}, fmt.Errorf("preflight: no version number found in %q", s) + } + major, _ := strconv.Atoi(m[1]) + minor, _ := strconv.Atoi(m[2]) + patch, _ := strconv.Atoi(m[3]) + return semver{major, minor, patch}, nil +} + +func (v semver) String() string { return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) } + +// Compare returns -1, 0, or 1 as v is less than, equal to, or greater than o. +func (v semver) Compare(o semver) int { + if v.Major != o.Major { + return cmp(v.Major, o.Major) + } + if v.Minor != o.Minor { + return cmp(v.Minor, o.Minor) + } + return cmp(v.Patch, o.Patch) +} + +func cmp(a, b int) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} + +// minSupportedSource is the oldest version this tool will start a migration +// from. Stalwart's own upgrade guidance says older installs need to reach +// 0.15.x first before crossing the 0.15/0.16 schema boundary this tool +// automates - see ARCHITECTURE.md §1/§4.1. +var minSupportedSource = semver{0, 15, 0} + +// DetectVersion runs the installed binary's --version flag and extracts a +// semver from its output. +func DetectVersion(ctx context.Context, binaryPath string) (string, error) { + cmd := exec.CommandContext(ctx, binaryPath, "--version") + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("preflight: run %s --version: %w (output: %s)", binaryPath, err, strings.TrimSpace(out.String())) + } + v, err := parseSemver(out.String()) + if err != nil { + return "", fmt.Errorf("preflight: parse version from %s --version output %q: %w", binaryPath, strings.TrimSpace(out.String()), err) + } + return v.String(), nil +} diff --git a/internal/preflight/version_test.go b/internal/preflight/version_test.go new file mode 100644 index 0000000..76fc051 --- /dev/null +++ b/internal/preflight/version_test.go @@ -0,0 +1,58 @@ +package preflight + +import "testing" + +func TestParseSemver(t *testing.T) { + cases := []struct { + in string + want string + wantErr bool + }{ + {"stalwart 0.15.5\n", "0.15.5", false}, + {"v0.16.14", "0.16.14", false}, + {"Stalwart Mail Server v0.16.0 (build abc123)", "0.16.0", false}, + {"no version here", "", true}, + {"", "", true}, + } + for _, tc := range cases { + got, err := parseSemver(tc.in) + if tc.wantErr { + if err == nil { + t.Errorf("parseSemver(%q) = %v, want error", tc.in, got) + } + continue + } + if err != nil { + t.Errorf("parseSemver(%q) unexpected error: %v", tc.in, err) + continue + } + if got.String() != tc.want { + t.Errorf("parseSemver(%q) = %s, want %s", tc.in, got, tc.want) + } + } +} + +func TestSemverCompare(t *testing.T) { + cases := []struct { + a, b string + want int + }{ + {"0.15.5", "0.16.0", -1}, + {"0.16.14", "0.16.14", 0}, + {"0.16.1", "0.15.5", 1}, + {"1.0.0", "0.16.14", 1}, + } + for _, tc := range cases { + a, err := parseSemver(tc.a) + if err != nil { + t.Fatalf("parseSemver(%q): %v", tc.a, err) + } + b, err := parseSemver(tc.b) + if err != nil { + t.Fatalf("parseSemver(%q): %v", tc.b, err) + } + if got := a.Compare(b); got != tc.want { + t.Errorf("%s.Compare(%s) = %d, want %d", tc.a, tc.b, got, tc.want) + } + } +} diff --git a/internal/recovery/apply.go b/internal/recovery/apply.go new file mode 100644 index 0000000..1742b1f --- /dev/null +++ b/internal/recovery/apply.go @@ -0,0 +1,54 @@ +package recovery + +import ( + "context" + "fmt" + "os" + "os/exec" +) + +// ApplyOptions configures stalwart-cli apply invocations against a +// recovery-mode instance - see UPGRADING/v0_16.md's documented +// STALWART_URL/STALWART_USER/STALWART_PASSWORD + `stalwart-cli apply --file` +// sequence. +type ApplyOptions struct { + CLIBinaryPath string // defaults to "stalwart-cli" + URL string + User string + Password string +} + +// Apply runs `stalwart-cli apply --file ` once, with credentials +// passed as environment variables exactly as the upgrade guide's own +// example does, rather than on the command line where they'd be visible to +// anything reading this process's argv. +func Apply(ctx context.Context, o ApplyOptions, file string) error { + binary := o.CLIBinaryPath + if binary == "" { + binary = "stalwart-cli" + } + cmd := exec.CommandContext(ctx, binary, "apply", "--file", file) + cmd.Env = append(os.Environ(), + "STALWART_URL="+o.URL, + "STALWART_USER="+o.User, + "STALWART_PASSWORD="+o.Password, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("recovery: stalwart-cli apply --file %s failed: %w (output: %s)", file, err, out) + } + return nil +} + +// ApplyAll runs Apply for each file in order - export.json first, then any +// additional test-deployment snapshots (ARCHITECTURE.md §4.3/§4.4) - +// stopping at the first failure so a partial, silently-incomplete settings +// replay never gets reported as success. +func ApplyAll(ctx context.Context, o ApplyOptions, files []string) error { + for i, f := range files { + if err := Apply(ctx, o, f); err != nil { + return fmt.Errorf("applied %d/%d file(s) before failing: %w", i, len(files), err) + } + } + return nil +} diff --git a/internal/recovery/apply_test.go b/internal/recovery/apply_test.go new file mode 100644 index 0000000..8a8d920 --- /dev/null +++ b/internal/recovery/apply_test.go @@ -0,0 +1,60 @@ +package recovery + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" +) + +func TestApplyInvokesCLIWithEnvCredentials(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + withFakeExecutable(t, "stalwart-cli", fmt.Sprintf("#!/bin/sh\necho \"$@ URL=$STALWART_URL USER=$STALWART_USER\" >> %q\nexit 0\n", log)) + + err := Apply(context.Background(), ApplyOptions{URL: "http://127.0.0.1:8080", User: "admin", Password: "secret"}, "/tmp/export.json") + if err != nil { + t.Fatalf("Apply: %v", err) + } + got := readArgsFile(t, log) + for _, want := range []string{"apply", "--file /tmp/export.json", "URL=http://127.0.0.1:8080", "USER=admin"} { + if !strings.Contains(got, want) { + t.Errorf("stalwart-cli invoked with %q, missing %q", got, want) + } + } + if strings.Contains(got, "secret") { + t.Error("password should not appear in argv") + } +} + +func TestApplyPropagatesFailure(t *testing.T) { + withFakeExecutable(t, "stalwart-cli", "#!/bin/sh\necho 'invalid object' >&2\nexit 1\n") + err := Apply(context.Background(), ApplyOptions{URL: "http://127.0.0.1:8080", User: "admin", Password: "x"}, "/tmp/export.json") + if err == nil { + t.Fatal("Apply should error when stalwart-cli exits non-zero") + } + if !strings.Contains(err.Error(), "invalid object") { + t.Errorf("error = %v, want it to include stderr", err) + } +} + +func TestApplyAllStopsAtFirstFailure(t *testing.T) { + dir := t.TempDir() + log := argsFile(t, dir) + script := fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\ncase \"$*\" in\n *bad.json*) exit 1 ;;\n *) exit 0 ;;\nesac\n", log) + withFakeExecutable(t, "stalwart-cli", script) + + err := ApplyAll(context.Background(), ApplyOptions{URL: "http://127.0.0.1:8080", User: "admin", Password: "x"}, + []string{filepath.Join(dir, "good1.json"), filepath.Join(dir, "bad.json"), filepath.Join(dir, "good2.json")}) + if err == nil { + t.Fatal("ApplyAll should fail on bad.json") + } + got := readArgsFile(t, log) + if strings.Contains(got, "good2.json") { + t.Error("ApplyAll should stop before applying good2.json after bad.json failed") + } + if !strings.Contains(err.Error(), "1/3") { + t.Errorf("error = %v, want it to report 1/3 files applied before failing", err) + } +} diff --git a/internal/recovery/doc.go b/internal/recovery/doc.go new file mode 100644 index 0000000..bbb631e --- /dev/null +++ b/internal/recovery/doc.go @@ -0,0 +1,3 @@ +// Package recovery implements supervision of the Stalwart recovery-mode process and the settings apply step. +// See ARCHITECTURE.md §4.4 for the design. +package recovery diff --git a/internal/recovery/exectest_test.go b/internal/recovery/exectest_test.go new file mode 100644 index 0000000..5519c4a --- /dev/null +++ b/internal/recovery/exectest_test.go @@ -0,0 +1,39 @@ +package recovery + +import ( + "os" + "path/filepath" + "testing" +) + +// withFakeExecutable puts a fake executable named `name` at the front of +// PATH for the duration of the test, so code that shells out to a +// real-world tool (stalwart-cli) can be exercised without that tool +// actually being installed. t.Setenv restores PATH automatically. +func withFakeExecutable(t *testing.T, name, script string) (dir string) { + t.Helper() + dir = t.TempDir() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + return dir +} + +func argsFile(t *testing.T, dir string) string { + t.Helper() + return filepath.Join(dir, "invoked-args.log") +} + +func readArgsFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return "" + } + if err != nil { + t.Fatal(err) + } + return string(data) +} diff --git a/internal/recovery/health.go b/internal/recovery/health.go new file mode 100644 index 0000000..07e961e --- /dev/null +++ b/internal/recovery/health.go @@ -0,0 +1,44 @@ +package recovery + +import ( + "context" + "fmt" + "net/http" + "time" +) + +// WaitForHealthy polls url until it responds (any status - even 401 proves +// the HTTP server itself is up and routing requests, which is what this +// check exists to confirm) or timeout elapses, returning a descriptive +// error on timeout rather than hanging indefinitely. This is what makes +// recovery mode's startup supervised rather than fire-and-forget - see +// ARCHITECTURE.md §4.4. +func WaitForHealthy(ctx context.Context, httpClient *http.Client, url string, timeout time.Duration) error { + if httpClient == nil { + httpClient = &http.Client{Timeout: 5 * time.Second} + } + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if err := ctx.Err(); err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := httpClient.Do(req) + if err != nil { + lastErr = err + } else { + resp.Body.Close() + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(250 * time.Millisecond): + } + } + return fmt.Errorf("recovery: %s did not become reachable within %s: %w", url, timeout, lastErr) +} diff --git a/internal/recovery/main_test.go b/internal/recovery/main_test.go new file mode 100644 index 0000000..444e685 --- /dev/null +++ b/internal/recovery/main_test.go @@ -0,0 +1,51 @@ +package recovery + +import ( + "fmt" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "testing" +) + +// TestMain lets this test binary also act as a fake Stalwart binary for +// subprocess tests - the standard os/exec "helper process" technique (see +// Go's own os/exec_test.go). When STALWART_MIGRATOR_TEST_HELPER=1 is set, +// the binary runs a minimal HTTP server on STALWART_MIGRATOR_TEST_PORT +// until it receives SIGTERM (or, if STALWART_MIGRATOR_TEST_IGNORE_SIGTERM=1, +// ignores SIGTERM to exercise the SIGKILL escalation path) instead of +// running the actual test suite. This avoids needing a real Stalwart binary +// - or a fabricated stand-in for its HTTP behavior - anywhere in these +// tests: it's real net/http and real process signaling, just running under +// this package's own compiled binary. +func TestMain(m *testing.M) { + if os.Getenv("STALWART_MIGRATOR_TEST_HELPER") == "1" { + runFakeStalwartServer() + return + } + os.Exit(m.Run()) +} + +func runFakeStalwartServer() { + port := os.Getenv("STALWART_MIGRATOR_TEST_PORT") + ln, err := net.Listen("tcp", "127.0.0.1:"+port) + if err != nil { + fmt.Fprintln(os.Stderr, "fake stalwart: listen:", err) + os.Exit(1) + } + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })} + go srv.Serve(ln) + + sigCh := make(chan os.Signal, 1) + if os.Getenv("STALWART_MIGRATOR_TEST_IGNORE_SIGTERM") == "1" { + signal.Ignore(syscall.SIGTERM) + select {} // block forever; the test must SIGKILL this process itself + } + signal.Notify(sigCh, syscall.SIGTERM) + <-sigCh + os.Exit(0) +} diff --git a/internal/recovery/process.go b/internal/recovery/process.go new file mode 100644 index 0000000..b850980 --- /dev/null +++ b/internal/recovery/process.go @@ -0,0 +1,96 @@ +package recovery + +import ( + "context" + "fmt" + "os" + "os/exec" + "syscall" + "time" +) + +// ProcessOptions configures how the target binary is launched. +type ProcessOptions struct { + BinaryPath string + ConfigPath string + + // RecoveryMode, when true, sets STALWART_RECOVERY_MODE=1 and + // STALWART_RECOVERY_ADMIN=: - exactly the + // environment variables Stalwart's own upgrade guide uses to bring the + // new binary up in recovery mode against a not-yet-migrated store (see + // UPGRADING/v0_16.md). When false, the binary is started as a normal + // boot against ConfigPath - used after recovery mode to confirm the + // migrated store comes up cleanly under an ordinary start, not just a + // recovery one (see ARCHITECTURE.md's dry-run design). + RecoveryMode bool + AdminUser string + AdminPassword string + + // ExtraEnv is appended after any recovery-mode env vars. This is what + // lets a dry-run point the process at a sandbox without RecoveryMode's + // two env vars becoming the only way to parameterize the child process. + ExtraEnv []string +} + +// Process supervises one run of the target Stalwart binary as a background +// child process, so a caller can start it, wait for it to become healthy, +// interact with it, and stop it again - without ever touching a real +// systemd unit or Docker container. See ARCHITECTURE.md §4.4. +type Process struct { + cmd *exec.Cmd +} + +// Start launches the binary. It returns as soon as the OS has started the +// process - it does not wait for Stalwart itself to become ready; use +// WaitForHealthy for that. +func (p *Process) Start(ctx context.Context, o ProcessOptions) error { + var env []string + if o.RecoveryMode { + env = append(env, + "STALWART_RECOVERY_MODE=1", + fmt.Sprintf("STALWART_RECOVERY_ADMIN=%s:%s", o.AdminUser, o.AdminPassword), + ) + } + env = append(env, o.ExtraEnv...) + + cmd := exec.CommandContext(ctx, o.BinaryPath, "--config", o.ConfigPath) + cmd.Env = append(os.Environ(), env...) + if err := cmd.Start(); err != nil { + return fmt.Errorf("recovery: start %s: %w", o.BinaryPath, err) + } + p.cmd = cmd + return nil +} + +// Stop sends SIGTERM and waits up to gracePeriod for the process to exit - +// mirroring the upgrade guide's own "Ctrl+C in the first terminal" step, +// just automated - escalating to SIGKILL if it doesn't exit in time so a +// stuck child process can never hang a migration run indefinitely. Safe to +// call on a Process that was never successfully started. +func (p *Process) Stop(gracePeriod time.Duration) error { + if p.cmd == nil || p.cmd.Process == nil { + return nil + } + if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil { + return fmt.Errorf("recovery: signal process (pid %d): %w", p.cmd.Process.Pid, err) + } + + done := make(chan error, 1) + go func() { done <- p.cmd.Wait() }() + + select { + case err := <-done: + // A non-zero exit from a SIGTERM-based shutdown is expected and not + // itself a failure worth reporting - only an unexpected Wait error is. + if err != nil { + if _, ok := err.(*exec.ExitError); !ok { + return fmt.Errorf("recovery: wait for process (pid %d): %w", p.cmd.Process.Pid, err) + } + } + return nil + case <-time.After(gracePeriod): + _ = p.cmd.Process.Kill() + <-done + return fmt.Errorf("recovery: process (pid %d) did not exit within %s of SIGTERM - sent SIGKILL", p.cmd.Process.Pid, gracePeriod) + } +} diff --git a/internal/recovery/process_test.go b/internal/recovery/process_test.go new file mode 100644 index 0000000..48f5fa6 --- /dev/null +++ b/internal/recovery/process_test.go @@ -0,0 +1,104 @@ +package recovery + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" +) + +// freePort asks the OS for an unused TCP port by binding to :0 and +// immediately releasing it. There's a small window where something else +// could grab it before the fake server binds, but that's an acceptable, +// standard tradeoff for tests. +func freePort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +func helperProcessEnv(port int, extra ...string) []string { + env := []string{ + "STALWART_MIGRATOR_TEST_HELPER=1", + fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port), + } + return append(env, extra...) +} + +func testBinaryPath(t *testing.T) string { + t.Helper() + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + return self +} + +func TestProcessStartWaitHealthyStop(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + proc := &Process{} + err := proc.Start(context.Background(), ProcessOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ExtraEnv: helperProcessEnv(port), + }) + if err != nil { + t.Fatalf("Start: %v", err) + } + + url := fmt.Sprintf("http://127.0.0.1:%d/", port) + if err := WaitForHealthy(context.Background(), nil, url, 5*time.Second); err != nil { + t.Fatalf("WaitForHealthy: %v", err) + } + + if err := proc.Stop(5 * time.Second); err != nil { + t.Fatalf("Stop: %v", err) + } +} + +func TestProcessStopEscalatesToSIGKILL(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + proc := &Process{} + err := proc.Start(context.Background(), ProcessOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ExtraEnv: helperProcessEnv(port, "STALWART_MIGRATOR_TEST_IGNORE_SIGTERM=1"), + }) + if err != nil { + t.Fatalf("Start: %v", err) + } + + url := fmt.Sprintf("http://127.0.0.1:%d/", port) + if err := WaitForHealthy(context.Background(), nil, url, 5*time.Second); err != nil { + t.Fatalf("WaitForHealthy: %v", err) + } + + err = proc.Stop(500 * time.Millisecond) + if err == nil { + t.Fatal("Stop should report an error when it had to escalate to SIGKILL") + } +} + +func TestWaitForHealthyTimesOut(t *testing.T) { + // Nothing listens on this port. + port := freePort(t) + url := fmt.Sprintf("http://127.0.0.1:%d/", port) + + err := WaitForHealthy(context.Background(), nil, url, 500*time.Millisecond) + if err == nil { + t.Fatal("WaitForHealthy should time out when nothing is listening") + } +} diff --git a/internal/recovery/recovery.go b/internal/recovery/recovery.go new file mode 100644 index 0000000..8289bd2 --- /dev/null +++ b/internal/recovery/recovery.go @@ -0,0 +1,107 @@ +package recovery + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net/http" + "time" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" +) + +// Options configures one full recovery-mode migration cycle: starting the +// target binary in recovery mode, waiting for it to come up, applying the +// settings snapshot(s), and stopping it again. See ARCHITECTURE.md §4.4. +type Options struct { + BinaryPath string + ConfigPath string + ListenURL string // recovery mode's own HTTP listener, e.g. "http://127.0.0.1:8080" + AdminUser string + ApplyFiles []string + CLIBinaryPath string + ExtraEnv []string // lets a dry-run point ports/paths at a sandbox without touching production config + StartupTimeout time.Duration + StopGrace time.Duration + HTTPClient *http.Client +} + +// GenerateRecoveryPassword returns a fresh random one-time password for +// STALWART_RECOVERY_ADMIN - never the operator's real admin password, never +// logged, never reused across runs or persisted to the checkpoint. +func GenerateRecoveryPassword() (string, error) { + b := make([]byte, 20) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("recovery: generate password: %w", err) + } + return hex.EncodeToString(b), nil +} + +// Run executes one recovery-mode cycle as a single checkpointed step. It is +// deliberately not decomposed into per-sub-step checkpoints the way +// preflight and backup are: if this tool's own process crashes mid-cycle, +// the child Stalwart process it started may or may not still be running +// independently, and blindly "resuming" by reattaching to a guessed PID or +// killing an unrelated process on the recovery port would be more dangerous +// than just retrying cleanly. A retry that hits "address already in use" +// surfaces the real problem (an orphaned process from the failed attempt) +// for a human to clear, rather than this tool guessing at cleanup. +// +// Whatever happens after Start succeeds, Stop is always attempted on the +// way out (via a deferred call), so a failure partway through this cycle +// doesn't leak the child process within a single invocation. +func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (Report, error) { + var report Report + + outcome, err := store.RunStep(rs, checkpoint.PhaseRecovery, "recovery-cycle", func() (out checkpoint.StepOutcome, err error) { + password, err := GenerateRecoveryPassword() + if err != nil { + return checkpoint.StepOutcome{}, err + } + + proc := &Process{} + if startErr := proc.Start(ctx, ProcessOptions{ + BinaryPath: opts.BinaryPath, ConfigPath: opts.ConfigPath, + RecoveryMode: true, AdminUser: opts.AdminUser, AdminPassword: password, + ExtraEnv: opts.ExtraEnv, + }); startErr != nil { + return checkpoint.StepOutcome{}, startErr + } + + stopGrace := opts.StopGrace + if stopGrace <= 0 { + stopGrace = 10 * time.Second + } + defer func() { + if stopErr := proc.Stop(stopGrace); stopErr != nil && err == nil { + err = stopErr + } + }() + + startupTimeout := opts.StartupTimeout + if startupTimeout <= 0 { + startupTimeout = 60 * time.Second + } + if healthErr := WaitForHealthy(ctx, opts.HTTPClient, opts.ListenURL, startupTimeout); healthErr != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("recovery mode did not come up: %w", healthErr) + } + + if applyErr := ApplyAll(ctx, ApplyOptions{ + CLIBinaryPath: opts.CLIBinaryPath, URL: opts.ListenURL, User: opts.AdminUser, Password: password, + }, opts.ApplyFiles); applyErr != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("settings apply failed: %w", applyErr) + } + + return checkpoint.StepOutcome{ + Detail: fmt.Sprintf("recovery mode came up at %s, applied %d settings file(s), stopped cleanly", opts.ListenURL, len(opts.ApplyFiles)), + }, nil + }) + + if err != nil { + report.Results = append(report.Results, CheckResult{Name: "recovery-cycle", Status: StatusFail, Detail: err.Error()}) + return report, err + } + report.Results = append(report.Results, CheckResult{Name: "recovery-cycle", Status: StatusOK, Detail: outcome.Detail}) + return report, nil +} diff --git a/internal/recovery/recovery_test.go b/internal/recovery/recovery_test.go new file mode 100644 index 0000000..4c9befd --- /dev/null +++ b/internal/recovery/recovery_test.go @@ -0,0 +1,113 @@ +package recovery + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" +) + +func TestRecoveryRunEndToEndAndResume(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + applyDir := t.TempDir() + applyLog := argsFile(t, applyDir) + withFakeExecutable(t, "stalwart-cli", fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\nexit 0\n", applyLog)) + + exportFile := filepath.Join(t.TempDir(), "export.json") + os.WriteFile(exportFile, []byte("{}"), 0o644) + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("store.Create: %v", err) + } + + opts := Options{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + AdminUser: "admin", + ApplyFiles: []string{exportFile}, + ExtraEnv: helperProcessEnv(port), + StartupTimeout: 5 * time.Second, + StopGrace: 5 * time.Second, + } + + report, err := Run(context.Background(), store, rs, opts) + if err != nil { + t.Fatalf("Run #1: %v", err) + } + if len(report.Results) != 1 || report.Results[0].Status != StatusOK { + t.Fatalf("Run #1 report = %+v, want a single OK result", report.Results) + } + if !rs.Done(checkpoint.PhaseRecovery, "recovery-cycle") { + t.Fatal("recovery-cycle should be marked done after a successful run") + } + if got := readArgsFile(t, applyLog); got == "" { + t.Fatal("stalwart-cli apply was never invoked") + } + + // -- resume: break the CLI so a re-invocation would fail loudly, and use + // -- a port nothing listens on, so re-starting the process would time out. + withFakeExecutable(t, "stalwart-cli", "#!/bin/sh\necho should-not-run-again >&2\nexit 1\n") + badPort := freePort(t) + resumedOpts := opts + resumedOpts.ListenURL = fmt.Sprintf("http://127.0.0.1:%d/", badPort) + resumedOpts.ExtraEnv = helperProcessEnv(badPort) + resumedOpts.StartupTimeout = 300 * time.Millisecond + + resumed, err := store.Load(rs.RunID) + if err != nil { + t.Fatalf("store.Load (resume): %v", err) + } + report2, err := Run(context.Background(), store, resumed, resumedOpts) + if err != nil { + t.Fatalf("Run #2 (resume) should succeed without redoing the cycle: %v", err) + } + if len(report2.Results) != 1 || report2.Results[0].Detail != report.Results[0].Detail { + t.Errorf("resumed report = %+v, want the cached outcome from Run #1", report2.Results) + } +} + +func TestRecoveryRunFailsWhenProcessNeverBecomesHealthy(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + // Nothing listens on this port - the binary never actually starts an + // HTTP server, since BinaryPath here is a script that just exits. + binDir := t.TempDir() + binPath := filepath.Join(binDir, "stalwart") + os.WriteFile(binPath, []byte("#!/bin/sh\nsleep 5\n"), 0o755) + port := freePort(t) + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatal(err) + } + + opts := Options{ + BinaryPath: binPath, + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + AdminUser: "admin", + ApplyFiles: []string{}, + StartupTimeout: 300 * time.Millisecond, + StopGrace: 2 * time.Second, + } + + _, err = Run(context.Background(), store, rs, opts) + if err == nil { + t.Fatal("Run should fail when the process never becomes healthy") + } + if rs.Status(checkpoint.PhaseRecovery, "recovery-cycle") != checkpoint.StepFailed { + t.Errorf("step status = %s, want failed (so a retry is possible)", rs.Status(checkpoint.PhaseRecovery, "recovery-cycle")) + } +} diff --git a/internal/recovery/report.go b/internal/recovery/report.go new file mode 100644 index 0000000..4fbcff6 --- /dev/null +++ b/internal/recovery/report.go @@ -0,0 +1,31 @@ +package recovery + +import ( + "fmt" + "strings" +) + +type Status string + +const ( + StatusOK Status = "ok" + StatusFail Status = "fail" +) + +type CheckResult struct { + Name string + Status Status + Detail string +} + +type Report struct { + Results []CheckResult +} + +func (r Report) String() string { + var b strings.Builder + for _, res := range r.Results { + fmt.Fprintf(&b, "[%-4s] %-16s %s\n", strings.ToUpper(string(res.Status)), res.Name, res.Detail) + } + return b.String() +} diff --git a/internal/rollback/doc.go b/internal/rollback/doc.go new file mode 100644 index 0000000..99f3baa --- /dev/null +++ b/internal/rollback/doc.go @@ -0,0 +1,3 @@ +// Package rollback implements restoring the pre-migration backup and old binary on failure. +// See ARCHITECTURE.md §4.8 for the design. +package rollback diff --git a/internal/stalwartapi/client.go b/internal/stalwartapi/client.go new file mode 100644 index 0000000..d81d6ba --- /dev/null +++ b/internal/stalwartapi/client.go @@ -0,0 +1,73 @@ +package stalwartapi + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" +) + +// Client is the shared JMAP/management-API client every migration phase +// uses to talk to a Stalwart instance. It stays deliberately thin: phases +// that need Stalwart-specific behavior (recovery-mode control, apply-plan +// replay, account/mailbox introspection) get methods added here only as +// their wire-level details are confirmed against Stalwart's actual source +// and documentation, never guessed - see management.go and mailbox.go for +// what that grounding looked like for account enumeration and mailbox +// counts respectively. +type Client struct { + BaseURL string // e.g. "https://mail.example.com" + Username string + Password string + HTTPClient *http.Client +} + +func (c *Client) httpClient() *http.Client { + if c.HTTPClient != nil { + return c.HTTPClient + } + return &http.Client{Timeout: 15 * time.Second} +} + +// Ping confirms the instance is reachable and the given credentials are +// accepted, via JMAP session discovery (RFC 8620 §2, the well-known +// /.well-known/jmap endpoint) over HTTP Basic auth. This is preflight's +// "dry-run" reachability check (ARCHITECTURE.md §4.1): it does nothing but +// read the session document, so it's safe to run against a live production +// server before anything else in the migration happens. +func (c *Client) Ping(ctx context.Context) error { + url := strings.TrimRight(c.BaseURL, "/") + "/.well-known/jmap" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + req.SetBasicAuth(c.Username, c.Password) + resp, err := c.httpClient().Do(req) + if err != nil { + return fmt.Errorf("stalwartapi: reach %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("stalwartapi: session request to %s returned %s", url, resp.Status) + } + return nil +} + +// Snapshot mirrors checkpoint.PreflightSnapshot's shape without this +// package depending on the checkpoint package's format. +type Snapshot struct { + AccountCount int + Domains []string + MailboxCounts map[string][]MailboxCount // account email -> its mailboxes + // MailboxErrors records, per account email, why that account's mailbox + // counts couldn't be captured (e.g. impersonation not permitted for + // that account). A non-empty entry here means MailboxCounts has no + // entry for that account - it's not silently treated as zero messages. + MailboxErrors map[string]string +} + +type MailboxCount struct { + Mailbox string + Messages int +} diff --git a/internal/stalwartapi/doc.go b/internal/stalwartapi/doc.go new file mode 100644 index 0000000..ace7e59 --- /dev/null +++ b/internal/stalwartapi/doc.go @@ -0,0 +1,3 @@ +// Package stalwartapi implements the JMAP and management-API client shared by every other phase. +// See ARCHITECTURE.md §7 for the design. +package stalwartapi diff --git a/internal/stalwartapi/mailbox.go b/internal/stalwartapi/mailbox.go new file mode 100644 index 0000000..05974c7 --- /dev/null +++ b/internal/stalwartapi/mailbox.go @@ -0,0 +1,112 @@ +package stalwartapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// jmapSession is the subset of RFC 8620 §2's Session object this package +// needs: apiUrl (where to POST standard JMAP method calls - a different +// endpoint from Stalwart's /api management API, confirmed in +// docs/ref/object/account.md) and primaryAccounts (which accountId the +// urn:ietf:params:jmap:mail capability maps to for the authenticated +// identity). +type jmapSession struct { + APIURL string `json:"apiUrl"` + PrimaryAccounts map[string]string `json:"primaryAccounts"` +} + +const jmapMailCapability = "urn:ietf:params:jmap:mail" + +// fetchSession performs JMAP session discovery (RFC 8620 §2, +// /.well-known/jmap) with the given credentials. +func (c *Client) fetchSession(ctx context.Context, username, password string) (*jmapSession, error) { + url := strings.TrimRight(c.BaseURL, "/") + "/.well-known/jmap" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(username, password) + resp, err := c.httpClient().Do(req) + if err != nil { + return nil, fmt.Errorf("session discovery at %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("session discovery at %s returned %s", url, resp.Status) + } + var session jmapSession + if err := json.NewDecoder(resp.Body).Decode(&session); err != nil { + return nil, fmt.Errorf("parse session document from %s: %w", url, err) + } + if session.APIURL == "" { + return nil, fmt.Errorf("session document from %s has no apiUrl", url) + } + return &session, nil +} + +type mailboxGetEntry struct { + Name string `json:"name"` + TotalEmails int `json:"totalEmails"` +} + +// MailboxSnapshot captures every mailbox's message count for one account, +// authenticating as that account via Stalwart's documented impersonation +// mechanism rather than assuming this Client's own credentials get direct +// cross-account access. That assumption would be wrong: Stalwart's JMAP +// session `accounts`/`primaryAccounts` map is built only from the +// authenticated identity's own membership and sharing grants - it is NOT +// expanded for a superuser (confirmed against +// crates/jmap/src/api/session.rs), so a plain Mailbox/get call for an +// arbitrary accountId under this Client's own login would be rejected. +// +// Instead, this Client's Username must hold Stalwart's `impersonate` +// permission (see docs/auth/authorization/administrator.md), and this +// method logs in AS the target account using the documented composite +// login format "%" with the impersonator's password, +// then calls standard RFC 8621 Mailbox/get - reading the exact wire +// property name `totalEmails` - against the URL that account's own JMAP +// session document reports as apiUrl (not the /api management endpoint +// x:Account/* uses; confirmed as a distinct endpoint in +// docs/ref/object/account.md). +func (c *Client) MailboxSnapshot(ctx context.Context, targetEmail string) ([]MailboxCount, error) { + impersonatedUser := fmt.Sprintf("%s%%%s", targetEmail, c.Username) + + session, err := c.fetchSession(ctx, impersonatedUser, c.Password) + if err != nil { + return nil, fmt.Errorf("stalwartapi: impersonate %s: %w", targetEmail, err) + } + accountID, ok := session.PrimaryAccounts[jmapMailCapability] + if !ok || accountID == "" { + return nil, fmt.Errorf("stalwartapi: impersonated session for %s has no %s account", targetEmail, jmapMailCapability) + } + + responses, err := c.callAs(ctx, impersonatedUser, c.Password, session.APIURL, + []string{"urn:ietf:params:jmap:core", jmapMailCapability}, + []any{[]any{"Mailbox/get", map[string]any{"accountId": accountID, "properties": []string{"name", "totalEmails"}}, "m"}}, + ) + if err != nil { + return nil, fmt.Errorf("stalwartapi: Mailbox/get for %s: %w", targetEmail, err) + } + if len(responses) == 0 { + return nil, fmt.Errorf("stalwartapi: Mailbox/get for %s returned no method responses", targetEmail) + } + if responses[0].Name == "error" { + return nil, fmt.Errorf("stalwartapi: Mailbox/get for %s error: %s", targetEmail, responses[0].Args) + } + + var result struct { + List []mailboxGetEntry `json:"list"` + } + if err := json.Unmarshal(responses[0].Args, &result); err != nil { + return nil, fmt.Errorf("stalwartapi: parse Mailbox/get response for %s: %w", targetEmail, err) + } + counts := make([]MailboxCount, len(result.List)) + for i, m := range result.List { + counts[i] = MailboxCount{Mailbox: m.Name, Messages: m.TotalEmails} + } + return counts, nil +} diff --git a/internal/stalwartapi/mailbox_test.go b/internal/stalwartapi/mailbox_test.go new file mode 100644 index 0000000..917b1b6 --- /dev/null +++ b/internal/stalwartapi/mailbox_test.go @@ -0,0 +1,112 @@ +package stalwartapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestMailboxSnapshotImpersonatesAndFetchesCounts(t *testing.T) { + var apiURL string + var sessionAuthUser, sessionAuthPass string + var gotAccountID string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap": + sessionAuthUser, sessionAuthPass, _ = r.BasicAuth() + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "primaryAccounts": map[string]string{jmapMailCapability: "mail-acct-1"}, + }) + case r.Method == http.MethodPost && r.URL.Path == "/jmap-api": + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + methodCalls := body["methodCalls"].([]any) + args := methodCalls[0].([]any)[1].(map[string]any) + gotAccountID, _ = args["accountId"].(string) + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"Mailbox/get", map[string]any{"list": []map[string]any{ + {"name": "Inbox", "totalEmails": 42}, + {"name": "Sent", "totalEmails": 7}, + }}, "m"}, + }}) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + apiURL = srv.URL + "/jmap-api" + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + counts, err := client.MailboxSnapshot(context.Background(), "alice@example.com") + if err != nil { + t.Fatalf("MailboxSnapshot: %v", err) + } + + if sessionAuthUser != "alice@example.com%admin" || sessionAuthPass != "hunter2" { + t.Errorf("session discovery auth = (%s, %s), want (alice@example.com%%admin, hunter2)", sessionAuthUser, sessionAuthPass) + } + if gotAccountID != "mail-acct-1" { + t.Errorf("Mailbox/get accountId = %s, want mail-acct-1", gotAccountID) + } + if len(counts) != 2 || counts[0].Mailbox != "Inbox" || counts[0].Messages != 42 || counts[1].Mailbox != "Sent" || counts[1].Messages != 7 { + t.Errorf("counts = %+v, want [{Inbox 42} {Sent 7}]", counts) + } +} + +func TestMailboxSnapshotFailsWhenImpersonationRejected(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "notasuperuser", Password: "x"} + _, err := client.MailboxSnapshot(context.Background(), "alice@example.com") + if err == nil { + t.Fatal("MailboxSnapshot should error when session discovery (impersonation) is rejected") + } +} + +func TestMailboxSnapshotFailsWhenSessionHasNoMailAccount(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": "http://unused/", + "primaryAccounts": map[string]string{}, + }) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + _, err := client.MailboxSnapshot(context.Background(), "alice@example.com") + if err == nil { + t.Fatal("MailboxSnapshot should error when the session has no jmap:mail primary account") + } +} + +func TestMailboxSnapshotPropagatesMailboxGetError(t *testing.T) { + var apiURL string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/.well-known/jmap": + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "primaryAccounts": map[string]string{jmapMailCapability: "mail-acct-1"}, + }) + case r.URL.Path == "/jmap-api": + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"error", map[string]any{"type": "accountNotFound"}, "m"}, + }}) + } + })) + defer srv.Close() + apiURL = srv.URL + "/jmap-api" + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + _, err := client.MailboxSnapshot(context.Background(), "alice@example.com") + if err == nil { + t.Fatal("MailboxSnapshot should propagate a JMAP-level error from Mailbox/get") + } +} diff --git a/internal/stalwartapi/management.go b/internal/stalwartapi/management.go new file mode 100644 index 0000000..594bec1 --- /dev/null +++ b/internal/stalwartapi/management.go @@ -0,0 +1,215 @@ +package stalwartapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" +) + +// managementCapabilities are the JMAP capability URNs Stalwart requires for +// its management object calls (x:Account/*): standard JMAP core plus its +// own urn:stalwart:jmap extension. Confirmed against +// docs/ref/object/account.md and crates/jmap-proto/src/request/capability.rs +// in stalwartlabs/stalwart. +var managementCapabilities = []string{"urn:ietf:params:jmap:core", "urn:stalwart:jmap"} + +type jmapRequest struct { + Using []string `json:"using"` + MethodCalls []any `json:"methodCalls"` +} + +type jmapRawResponse struct { + MethodResponses []json.RawMessage `json:"methodResponses"` +} + +// methodResponse is one [name, args, callId] triple from a JMAP response, +// per RFC 8620 §3.2 - Stalwart's management API follows the same envelope +// shape as its regular JMAP methods. +type methodResponse struct { + Name string + Args json.RawMessage + CallID string +} + +// call POSTs one JMAP-style request to the management API (Stalwart's /api +// endpoint - see docs/ref/object/account.md, which is distinct from /jmap) +// using this Client's own credentials, and returns its parsed method +// responses in order. +func (c *Client) call(ctx context.Context, using []string, methodCalls []any) ([]methodResponse, error) { + return c.callAs(ctx, c.Username, c.Password, strings.TrimRight(c.BaseURL, "/")+"/api", using, methodCalls) +} + +// callAs is call's underlying primitive: it accepts an explicit +// username/password/URL rather than always using this Client's own +// credentials and the management endpoint. MailboxSnapshot uses this to +// call standard JMAP methods (not Stalwart's x: management objects) against +// the URL a JMAP session document says to use, authenticated as an +// impersonated identity rather than this Client's own. +func (c *Client) callAs(ctx context.Context, username, password, url string, using []string, methodCalls []any) ([]methodResponse, error) { + reqBody, err := json.Marshal(jmapRequest{Using: using, MethodCalls: methodCalls}) + if err != nil { + return nil, fmt.Errorf("stalwartapi: encode request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(username, password) + + resp, err := c.httpClient().Do(req) + if err != nil { + return nil, fmt.Errorf("stalwartapi: call %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("stalwartapi: %s returned %s: %s", url, resp.Status, strings.TrimSpace(string(body))) + } + + var raw jmapRawResponse + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return nil, fmt.Errorf("stalwartapi: decode response from %s: %w", url, err) + } + responses := make([]methodResponse, 0, len(raw.MethodResponses)) + for _, r := range raw.MethodResponses { + var triple [3]json.RawMessage + if err := json.Unmarshal(r, &triple); err != nil { + return nil, fmt.Errorf("stalwartapi: parse method response envelope: %w", err) + } + var name, callID string + if err := json.Unmarshal(triple[0], &name); err != nil { + return nil, fmt.Errorf("stalwartapi: parse method response name: %w", err) + } + if err := json.Unmarshal(triple[2], &callID); err != nil { + return nil, fmt.Errorf("stalwartapi: parse method response call id: %w", err) + } + responses = append(responses, methodResponse{Name: name, Args: triple[1], CallID: callID}) + } + return responses, nil +} + +// account is the subset of x:Account/get's response fields this tool needs, +// confirmed against Stalwart's own docs/ref/object/account.md (whose +// stalwart-cli example is `query Account --fields id,name,domainId,usedDiskQuota`). +type account struct { + ID string `json:"id"` + Name string `json:"name"` + DomainID string `json:"domainId"` +} + +// AccountSnapshot enumerates every account on the instance via Stalwart's +// management API - x:Account/query to list ids, then x:Account/get to fetch +// their name/domainId - and returns the account count, set of domains in +// use, and (via MailboxSnapshot, per account) every mailbox's message +// count. This is what preflight's snapshot (ARCHITECTURE.md §4.1) and +// validate's directory-integrity and content-integrity checks (§4.7) +// compare before and after migration - the latter being the actual +// no-data-loss guarantee. +// +// A per-account mailbox-count failure (most likely: this Client's Username +// lacks the `impersonate` permission MailboxSnapshot depends on) does not +// fail the whole snapshot - the account/domain enumeration above is already +// useful on its own, and one account's failure shouldn't hide a working +// result for every other account. Instead it's recorded in +// Snapshot.MailboxErrors, keyed by account email, so callers can report +// exactly what's missing rather than silently treating an unreachable +// account's mailboxes as having zero messages. +func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { + queryResp, err := c.call(ctx, managementCapabilities, []any{ + []any{"x:Account/query", map[string]any{"filter": map[string]any{}}, "q"}, + }) + if err != nil { + return nil, fmt.Errorf("stalwartapi: Account/query: %w", err) + } + ids, err := accountQueryIDs(queryResp) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return &Snapshot{}, nil + } + + getResp, err := c.call(ctx, managementCapabilities, []any{ + []any{"x:Account/get", map[string]any{"ids": ids, "properties": []string{"id", "name", "domainId"}}, "g"}, + }) + if err != nil { + return nil, fmt.Errorf("stalwartapi: Account/get: %w", err) + } + accounts, err := accountGetList(getResp) + if err != nil { + return nil, err + } + + mailboxCounts := map[string][]MailboxCount{} + mailboxErrors := map[string]string{} + for _, a := range accounts { + if a.Name == "" { + continue // no login/email to impersonate against + } + counts, err := c.MailboxSnapshot(ctx, a.Name) + if err != nil { + mailboxErrors[a.Name] = err.Error() + continue + } + mailboxCounts[a.Name] = counts + } + + domainSet := map[string]bool{} + for _, a := range accounts { + if a.DomainID != "" { + domainSet[a.DomainID] = true + } + } + domains := make([]string, 0, len(domainSet)) + for d := range domainSet { + domains = append(domains, d) + } + sort.Strings(domains) + + return &Snapshot{ + AccountCount: len(accounts), + Domains: domains, + MailboxCounts: mailboxCounts, + MailboxErrors: mailboxErrors, + }, nil +} + +func accountQueryIDs(responses []methodResponse) ([]string, error) { + if len(responses) == 0 { + return nil, fmt.Errorf("stalwartapi: Account/query returned no method responses") + } + r := responses[0] + if r.Name == "error" { + return nil, fmt.Errorf("stalwartapi: Account/query error: %s", r.Args) + } + var result struct { + IDs []string `json:"ids"` + } + if err := json.Unmarshal(r.Args, &result); err != nil { + return nil, fmt.Errorf("stalwartapi: parse Account/query response: %w", err) + } + return result.IDs, nil +} + +func accountGetList(responses []methodResponse) ([]account, error) { + if len(responses) == 0 { + return nil, fmt.Errorf("stalwartapi: Account/get returned no method responses") + } + r := responses[0] + if r.Name == "error" { + return nil, fmt.Errorf("stalwartapi: Account/get error: %s", r.Args) + } + var result struct { + List []account `json:"list"` + } + if err := json.Unmarshal(r.Args, &result); err != nil { + return nil, fmt.Errorf("stalwartapi: parse Account/get response: %w", err) + } + return result.List, nil +} diff --git a/internal/stalwartapi/management_test.go b/internal/stalwartapi/management_test.go new file mode 100644 index 0000000..01c4b8b --- /dev/null +++ b/internal/stalwartapi/management_test.go @@ -0,0 +1,201 @@ +package stalwartapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// jmapEnvelope mirrors the wire shape this package's call() parses: a +// top-level {"methodResponses": [...]} object where each entry is a +// [name, args, callId] triple (RFC 8620 §3.2). +type jmapEnvelope struct { + MethodResponses []any `json:"methodResponses"` +} + +// accountManagementAndMailboxServer builds a fake server that answers both +// the x:Account/* management calls AccountSnapshot makes directly, and the +// session-discovery + Mailbox/get calls it makes indirectly (per account) +// via MailboxSnapshot. mailboxesFor maps an account email to the mailbox +// list its Mailbox/get should return; an account absent from the map gets a +// 403 on session discovery, simulating a missing `impersonate` grant. +func accountManagementAndMailboxServer(t *testing.T, mailboxesFor map[string][]map[string]any) (*httptest.Server, *[]string) { + t.Helper() + var gotPaths []string + var apiURL string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPaths = append(gotPaths, r.URL.Path) + + if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" { + user, _, _ := r.BasicAuth() + target := strings.SplitN(user, "%", 2)[0] + if _, ok := mailboxesFor[target]; !ok { + w.WriteHeader(http.StatusForbidden) + return + } + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "primaryAccounts": map[string]string{jmapMailCapability: "mail-" + target}, + }) + return + } + + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + methodCalls := body["methodCalls"].([]any) + first := methodCalls[0].([]any) + methodName := first[0].(string) + + switch methodName { + case "x:Account/query": + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"x:Account/query", map[string]any{"ids": []string{"a1", "a2"}}, "q"}, + }}) + case "x:Account/get": + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"x:Account/get", map[string]any{"list": []map[string]any{ + {"id": "a1", "name": "alice@example.com", "domainId": "example.com"}, + {"id": "a2", "name": "bob@example.org", "domainId": "example.org"}, + }}, "g"}, + }}) + case "Mailbox/get": + args := first[1].(map[string]any) + accountID := args["accountId"].(string) + target := strings.TrimPrefix(accountID, "mail-") + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"Mailbox/get", map[string]any{"list": mailboxesFor[target]}, "m"}, + }}) + default: + t.Errorf("unexpected method call: %s", methodName) + } + })) + apiURL = srv.URL + "/api" + return srv, &gotPaths +} + +func TestAccountSnapshotQueriesThenGets(t *testing.T) { + srv, _ := accountManagementAndMailboxServer(t, map[string][]map[string]any{ + "alice@example.com": {{"name": "Inbox", "totalEmails": 10}}, + "bob@example.org": {{"name": "Inbox", "totalEmails": 3}, {"name": "Archive", "totalEmails": 100}}, + }) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatalf("AccountSnapshot: %v", err) + } + if snap.AccountCount != 2 { + t.Errorf("AccountCount = %d, want 2", snap.AccountCount) + } + if len(snap.Domains) != 2 || snap.Domains[0] != "example.com" || snap.Domains[1] != "example.org" { + t.Errorf("Domains = %v, want [example.com example.org] (sorted)", snap.Domains) + } + if len(snap.MailboxErrors) != 0 { + t.Errorf("MailboxErrors = %v, want none (both accounts should succeed)", snap.MailboxErrors) + } + alice := snap.MailboxCounts["alice@example.com"] + if len(alice) != 1 || alice[0].Mailbox != "Inbox" || alice[0].Messages != 10 { + t.Errorf("alice's mailboxes = %+v, want [{Inbox 10}]", alice) + } + bob := snap.MailboxCounts["bob@example.org"] + if len(bob) != 2 || bob[1].Mailbox != "Archive" || bob[1].Messages != 100 { + t.Errorf("bob's mailboxes = %+v, want Inbox then Archive(100)", bob) + } +} + +func TestAccountSnapshotRecordsPerAccountMailboxFailureWithoutFailingOverall(t *testing.T) { + // bob@example.org is deliberately absent from mailboxesFor, simulating + // a missing `impersonate` grant for that one account. + srv, _ := accountManagementAndMailboxServer(t, map[string][]map[string]any{ + "alice@example.com": {{"name": "Inbox", "totalEmails": 10}}, + }) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatalf("AccountSnapshot should not fail overall just because one account's mailbox capture failed: %v", err) + } + if snap.AccountCount != 2 { + t.Errorf("AccountCount = %d, want 2 (account enumeration is unaffected by the mailbox-capture failure)", snap.AccountCount) + } + if _, ok := snap.MailboxCounts["alice@example.com"]; !ok { + t.Error("alice's mailbox counts should still be captured") + } + if _, ok := snap.MailboxCounts["bob@example.org"]; ok { + t.Error("bob's mailbox counts should NOT be present - his capture failed") + } + if _, ok := snap.MailboxErrors["bob@example.org"]; !ok { + t.Error("bob's failure should be recorded in MailboxErrors, not silently dropped") + } +} + +func TestAccountSnapshotEmptyInstance(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"x:Account/query", map[string]any{"ids": []string{}}, "q"}, + }}) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatalf("AccountSnapshot: %v", err) + } + if snap.AccountCount != 0 { + t.Errorf("AccountCount = %d, want 0", snap.AccountCount) + } +} + +func TestAccountSnapshotPropagatesJMAPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"error", map[string]any{"type": "forbidden"}, "q"}, + }}) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + _, err := client.AccountSnapshot(context.Background()) + if err == nil { + t.Fatal("AccountSnapshot should surface a JMAP-level error response") + } +} + +func TestAccountSnapshotPropagatesHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("invalid credentials")) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "wrong"} + _, err := client.AccountSnapshot(context.Background()) + if err == nil { + t.Fatal("AccountSnapshot should error on a non-200 response") + } +} + +func TestAccountSnapshotSendsBasicAuth(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if !ok || user != "admin" || pass != "hunter2" { + t.Errorf("BasicAuth = (%s, %s, %v), want (admin, hunter2, true)", user, pass, ok) + } + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"x:Account/query", map[string]any{"ids": []string{}}, "q"}, + }}) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + if _, err := client.AccountSnapshot(context.Background()); err != nil { + t.Fatalf("AccountSnapshot: %v", err) + } +} diff --git a/internal/validate/bootcheck.go b/internal/validate/bootcheck.go new file mode 100644 index 0000000..d242301 --- /dev/null +++ b/internal/validate/bootcheck.go @@ -0,0 +1,119 @@ +package validate + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" + "github.com/johnellis/stalwart-migrator/internal/recovery" + "github.com/johnellis/stalwart-migrator/internal/stalwartapi" +) + +// BootCheckOptions configures a normal (non-recovery-mode) boot of the +// migrated instance, to confirm it comes up cleanly outside recovery mode - +// not just that recovery mode itself could apply settings to it - and, +// optionally, a content-integrity comparison performed against that same +// boot before it's stopped again. +type BootCheckOptions struct { + BinaryPath string + ConfigPath string + ListenURL string + ExtraEnv []string + Timeout time.Duration + StopGrace time.Duration + HTTPClient *http.Client + + // ContentIntegrityBefore, if non-nil, is the pre-migration snapshot + // preflight captured (checkpoint.RunState.PreflightSnapshot). When set, + // BootCheck captures a fresh snapshot from the instance it just booted + // - authenticating with AdminUser/AdminPassword, which migrate over + // unchanged with the account (they don't need to differ from the + // pre-migration admin credentials) - and compares the two: this is the + // actual no-data-loss guarantee from ARCHITECTURE.md §4.7, not just + // "the migration mechanics ran". Left nil, only the boot-reachability + // check runs, e.g. when preflight never captured a snapshot because + // --admin-url wasn't set. + ContentIntegrityBefore *checkpoint.PreflightSnapshot + AdminUser string + AdminPassword string +} + +// BootCheck starts the target binary the way cutover eventually will (an +// ordinary boot, no STALWART_RECOVERY_MODE), waits for its HTTP listener to +// answer, optionally compares its content against ContentIntegrityBefore +// while it's up, then stops it. It reuses recovery.Process and +// recovery.WaitForHealthy rather than re-implementing process supervision, +// since "start the binary and confirm it's reachable" is exactly what those +// already do. +// +// Like recovery.Run, this is deliberately one atomic operation rather than +// separately checkpointed sub-steps: if this tool's own process crashes +// between the boot succeeding and the content check running, there's no +// safe way to reattach to whatever's left of the child process on resume, +// so a retry just redoes the whole cycle - see recovery.Run's doc comment +// for the full reasoning, which applies identically here. +func BootCheck(ctx context.Context, o BootCheckOptions) (detail string, result *ContentIntegrityResult, err error) { + proc := &recovery.Process{} + if startErr := proc.Start(ctx, recovery.ProcessOptions{ + BinaryPath: o.BinaryPath, ConfigPath: o.ConfigPath, RecoveryMode: false, ExtraEnv: o.ExtraEnv, + }); startErr != nil { + return "", nil, fmt.Errorf("validate: start normal boot: %w", startErr) + } + + stopGrace := o.StopGrace + if stopGrace <= 0 { + stopGrace = 10 * time.Second + } + defer func() { + if stopErr := proc.Stop(stopGrace); stopErr != nil && err == nil { + err = stopErr + } + }() + + timeout := o.Timeout + if timeout <= 0 { + timeout = 30 * time.Second + } + if healthErr := recovery.WaitForHealthy(ctx, o.HTTPClient, o.ListenURL, timeout); healthErr != nil { + return "", nil, fmt.Errorf("migrated instance did not come up under a normal (non-recovery-mode) boot: %w", healthErr) + } + detail = fmt.Sprintf("migrated instance booted normally (not in recovery mode) and answered at %s", o.ListenURL) + + if o.ContentIntegrityBefore == nil { + return detail, nil, nil + } + + client := &stalwartapi.Client{BaseURL: o.ListenURL, Username: o.AdminUser, Password: o.AdminPassword, HTTPClient: o.HTTPClient} + result, ciErr := compareContentIntegrity(ctx, client, o.ContentIntegrityBefore) + if ciErr != nil { + return detail, nil, fmt.Errorf("content-integrity comparison failed: %w", ciErr) + } + if !result.OK() { + return detail, result, fmt.Errorf("content integrity check found problems: %s", result.String()) + } + return detail, result, nil +} + +// Run executes BootCheck as a single checkpointed step, mirroring +// preflight/backup/recovery's pattern. +func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts BootCheckOptions) (Report, error) { + var report Report + outcome, err := store.RunStep(rs, checkpoint.PhaseValidate, "boot-check", func() (checkpoint.StepOutcome, error) { + detail, result, err := BootCheck(ctx, opts) + if err != nil { + return checkpoint.StepOutcome{}, err + } + if result != nil { + detail += " - " + result.String() + } + return checkpoint.StepOutcome{Detail: detail}, nil + }) + if err != nil { + report.Results = append(report.Results, CheckResult{Name: "boot-check", Status: StatusFail, Detail: err.Error()}) + return report, err + } + report.Results = append(report.Results, CheckResult{Name: "boot-check", Status: StatusOK, Detail: outcome.Detail}) + return report, nil +} diff --git a/internal/validate/bootcheck_test.go b/internal/validate/bootcheck_test.go new file mode 100644 index 0000000..914149f --- /dev/null +++ b/internal/validate/bootcheck_test.go @@ -0,0 +1,251 @@ +package validate + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" +) + +func freePort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +func testBinaryPath(t *testing.T) string { + t.Helper() + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + return self +} + +func TestBootCheckSucceedsWhenInstanceComesUp(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + detail, result, err := BootCheck(context.Background(), BootCheckOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + ExtraEnv: []string{ + "STALWART_MIGRATOR_TEST_HELPER=1", + fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port), + }, + Timeout: 5 * time.Second, + StopGrace: 5 * time.Second, + }) + if err != nil { + t.Fatalf("BootCheck: %v", err) + } + if detail == "" { + t.Error("BootCheck returned an empty detail on success") + } + if result != nil { + t.Errorf("result = %+v, want nil when ContentIntegrityBefore wasn't set", result) + } +} + +func TestBootCheckFailsWhenInstanceNeverComesUp(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + binPath := filepath.Join(t.TempDir(), "stalwart") + os.WriteFile(binPath, []byte("#!/bin/sh\nsleep 5\n"), 0o755) + port := freePort(t) + + _, _, err := BootCheck(context.Background(), BootCheckOptions{ + BinaryPath: binPath, + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + Timeout: 300 * time.Millisecond, + StopGrace: 2 * time.Second, + }) + if err == nil { + t.Fatal("BootCheck should fail when nothing ever answers ListenURL") + } +} + +func TestRunEndToEndAndResume(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatal(err) + } + + opts := BootCheckOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + ExtraEnv: []string{ + "STALWART_MIGRATOR_TEST_HELPER=1", + fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port), + }, + Timeout: 5 * time.Second, + StopGrace: 5 * time.Second, + } + + report, err := Run(context.Background(), store, rs, opts) + if err != nil { + t.Fatalf("Run #1: %v", err) + } + if report.Blocking() { + t.Fatalf("Run #1: unexpected failure: %s", report.String()) + } + + // Resume with a config that would fail if re-executed (nothing listens + // on badPort) - a skip proves the step didn't re-run. + badPort := freePort(t) + resumedOpts := opts + resumedOpts.ListenURL = fmt.Sprintf("http://127.0.0.1:%d/", badPort) + resumedOpts.Timeout = 300 * time.Millisecond + + resumed, err := store.Load(rs.RunID) + if err != nil { + t.Fatal(err) + } + report2, err := Run(context.Background(), store, resumed, resumedOpts) + if err != nil { + t.Fatalf("Run #2 (resume) should succeed without redoing the check: %v", err) + } + if report2.Blocking() { + t.Fatalf("Run #2 (resume): unexpected failure: %s", report2.String()) + } +} + +// beforeSnapshotWithAliceInbox builds a checkpoint.PreflightSnapshot +// matching the fake server's single hardcoded account (alice@example.com, +// mailbox "Inbox") with the given pre-migration message count. +func beforeSnapshotWithAliceInbox(messages int) *checkpoint.PreflightSnapshot { + return &checkpoint.PreflightSnapshot{ + AccountCount: 1, + Domains: []string{"example.com"}, + MailboxCounts: map[string][]checkpoint.MailboxCount{ + "alice@example.com": {{Mailbox: "Inbox", Messages: messages}}, + }, + } +} + +func TestBootCheckContentIntegrityPassesWhenCountsMatch(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + detail, result, err := BootCheck(context.Background(), BootCheckOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + ExtraEnv: []string{ + "STALWART_MIGRATOR_TEST_HELPER=1", + fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port), + "STALWART_MIGRATOR_TEST_MAILBOX_COUNT=42", // matches beforeSnapshotWithAliceInbox(42) + }, + Timeout: 5 * time.Second, + StopGrace: 5 * time.Second, + ContentIntegrityBefore: beforeSnapshotWithAliceInbox(42), + AdminUser: "admin", + AdminPassword: "hunter2", + }) + if err != nil { + t.Fatalf("BootCheck: %v", err) + } + if result == nil { + t.Fatal("result should be populated when ContentIntegrityBefore was set") + } + if !result.OK() { + t.Errorf("result.OK() = false, want true: %s", result.String()) + } + if result.AccountsChecked != 1 || result.MailboxesChecked != 1 { + t.Errorf("AccountsChecked=%d MailboxesChecked=%d, want 1 and 1", result.AccountsChecked, result.MailboxesChecked) + } + if detail == "" { + t.Error("detail should still describe the boot") + } +} + +func TestBootCheckContentIntegrityFailsWhenCountsMismatch(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + _, result, err := BootCheck(context.Background(), BootCheckOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + ExtraEnv: []string{ + "STALWART_MIGRATOR_TEST_HELPER=1", + fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port), + "STALWART_MIGRATOR_TEST_MAILBOX_COUNT=40", // the "after" server reports 40 + }, + Timeout: 5 * time.Second, + StopGrace: 5 * time.Second, + ContentIntegrityBefore: beforeSnapshotWithAliceInbox(42), // but "before" said 42 - two messages went missing + AdminUser: "admin", + AdminPassword: "hunter2", + }) + if err == nil { + t.Fatal("BootCheck should fail when a post-migration mailbox count doesn't match the pre-migration one") + } + if result == nil || result.OK() { + t.Fatalf("result = %+v, want a non-OK result describing the mismatch", result) + } + if len(result.MessageCountMismatches) != 1 { + t.Fatalf("MessageCountMismatches = %+v, want exactly one entry", result.MessageCountMismatches) + } + mismatch := result.MessageCountMismatches[0] + if mismatch.Account != "alice@example.com" || mismatch.Mailbox != "Inbox" || mismatch.Before != 42 || mismatch.After != 40 { + t.Errorf("mismatch = %+v, want alice@example.com/Inbox 42->40", mismatch) + } +} + +func TestBootCheckContentIntegrityDetectsMissingAccount(t *testing.T) { + port := freePort(t) + configPath := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(configPath, []byte("{}"), 0o644) + + before := &checkpoint.PreflightSnapshot{ + AccountCount: 2, + Domains: []string{"example.com", "example.net"}, + MailboxCounts: map[string][]checkpoint.MailboxCount{ + "alice@example.com": {{Mailbox: "Inbox", Messages: 42}}, + "carol@example.net": {{Mailbox: "Inbox", Messages: 5}}, // the fake server only ever knows about alice + }, + } + + _, result, err := BootCheck(context.Background(), BootCheckOptions{ + BinaryPath: testBinaryPath(t), + ConfigPath: configPath, + ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port), + ExtraEnv: []string{ + "STALWART_MIGRATOR_TEST_HELPER=1", + fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port), + "STALWART_MIGRATOR_TEST_MAILBOX_COUNT=42", + }, + Timeout: 5 * time.Second, + StopGrace: 5 * time.Second, + ContentIntegrityBefore: before, + AdminUser: "admin", + AdminPassword: "hunter2", + }) + if err == nil { + t.Fatal("BootCheck should fail when an account present before migration can't be found afterward") + } + if result == nil || len(result.MissingAccounts) != 1 || result.MissingAccounts[0] != "carol@example.net" { + t.Fatalf("result = %+v, want MissingAccounts = [carol@example.net]", result) + } +} diff --git a/internal/validate/content_integrity.go b/internal/validate/content_integrity.go new file mode 100644 index 0000000..8bacf7e --- /dev/null +++ b/internal/validate/content_integrity.go @@ -0,0 +1,113 @@ +package validate + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" + "github.com/johnellis/stalwart-migrator/internal/stalwartapi" +) + +// MailboxDelta is one mailbox whose message count didn't match between the +// pre- and post-migration snapshots. +type MailboxDelta struct { + Account string + Mailbox string + Before int + After int +} + +// ContentIntegrityResult is the outcome of comparing a pre-migration +// snapshot against a freshly captured post-migration one - the actual +// no-data-loss check described in ARCHITECTURE.md §4.7. +type ContentIntegrityResult struct { + AccountsChecked int + MailboxesChecked int + MissingAccounts []string // present before, not found after (even accounting for the email-address rewrite) + MessageCountMismatches []MailboxDelta // present both before and after, but with a different message count +} + +// OK reports whether every account and mailbox the pre-migration snapshot +// knew about was found afterward with an identical message count. +func (r ContentIntegrityResult) OK() bool { + return len(r.MissingAccounts) == 0 && len(r.MessageCountMismatches) == 0 +} + +func (r ContentIntegrityResult) String() string { + if r.OK() { + return fmt.Sprintf("content integrity: %d account(s), %d mailbox(es) checked, all message counts match", r.AccountsChecked, r.MailboxesChecked) + } + var b strings.Builder + fmt.Fprintf(&b, "content integrity: %d account(s), %d mailbox(es) checked", r.AccountsChecked, r.MailboxesChecked) + for _, a := range r.MissingAccounts { + fmt.Fprintf(&b, "; MISSING ACCOUNT %s", a) + } + for _, d := range r.MessageCountMismatches { + fmt.Fprintf(&b, "; MESSAGE COUNT MISMATCH %s/%s: %d before, %d after", d.Account, d.Mailbox, d.Before, d.After) + } + return b.String() +} + +// compareContentIntegrity captures a fresh snapshot via client and compares +// it against before, matching accounts by exact name first and falling +// back to the local part (the text before "@") since Stalwart's v0.16 +// migration rewrites bare usernames to full email addresses +// (UPGRADING/v0_16.md: "the migration script automatically assigns the +// default domain to accounts lacking one") - an exact-string comparison +// alone would misreport every rewritten account as missing. +func compareContentIntegrity(ctx context.Context, client *stalwartapi.Client, before *checkpoint.PreflightSnapshot) (*ContentIntegrityResult, error) { + after, err := client.AccountSnapshot(ctx) + if err != nil { + return nil, fmt.Errorf("capture post-migration snapshot: %w", err) + } + + result := &ContentIntegrityResult{} + + beforeAccounts := make([]string, 0, len(before.MailboxCounts)) + for a := range before.MailboxCounts { + beforeAccounts = append(beforeAccounts, a) + } + sort.Strings(beforeAccounts) + + for _, beforeAccount := range beforeAccounts { + result.AccountsChecked++ + afterMailboxes, found := after.MailboxCounts[beforeAccount] + if !found { + afterMailboxes, found = findByLocalPart(after.MailboxCounts, beforeAccount) + } + if !found { + result.MissingAccounts = append(result.MissingAccounts, beforeAccount) + continue + } + + afterByName := make(map[string]int, len(afterMailboxes)) + for _, m := range afterMailboxes { + afterByName[m.Mailbox] = m.Messages + } + + beforeMailboxes := append([]checkpoint.MailboxCount(nil), before.MailboxCounts[beforeAccount]...) + sort.Slice(beforeMailboxes, func(i, j int) bool { return beforeMailboxes[i].Mailbox < beforeMailboxes[j].Mailbox }) + for _, bm := range beforeMailboxes { + result.MailboxesChecked++ + afterCount, ok := afterByName[bm.Mailbox] + if !ok || afterCount != bm.Messages { + result.MessageCountMismatches = append(result.MessageCountMismatches, MailboxDelta{ + Account: beforeAccount, Mailbox: bm.Mailbox, Before: bm.Messages, After: afterCount, + }) + } + } + } + return result, nil +} + +func findByLocalPart(mailboxCounts map[string][]stalwartapi.MailboxCount, beforeAccount string) ([]stalwartapi.MailboxCount, bool) { + local := strings.SplitN(beforeAccount, "@", 2)[0] + for afterAccount, mb := range mailboxCounts { + if strings.SplitN(afterAccount, "@", 2)[0] == local { + return mb, true + } + } + return nil, false +} diff --git a/internal/validate/content_integrity_test.go b/internal/validate/content_integrity_test.go new file mode 100644 index 0000000..cd94e44 --- /dev/null +++ b/internal/validate/content_integrity_test.go @@ -0,0 +1,165 @@ +package validate + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/johnellis/stalwart-migrator/internal/checkpoint" + "github.com/johnellis/stalwart-migrator/internal/stalwartapi" +) + +// jmapEnvelope mirrors the wire shape stalwartapi.Client.call() parses. +type jmapEnvelope struct { + MethodResponses []any `json:"methodResponses"` +} + +// fakeManagementServer serves x:Account/query + x:Account/get from +// accounts, and, for each of them, session discovery + Mailbox/get from +// mailboxesByEmail (keyed by the account's post-migration email). +func fakeManagementServer(t *testing.T, accounts []map[string]any, mailboxesByEmail map[string][]map[string]any) *httptest.Server { + t.Helper() + var apiURL string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" { + user, _, _ := r.BasicAuth() + target := strings.SplitN(user, "%", 2)[0] + if _, ok := mailboxesByEmail[target]; !ok { + w.WriteHeader(http.StatusForbidden) + return + } + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "primaryAccounts": map[string]string{"urn:ietf:params:jmap:mail": "mail-" + target}, + }) + return + } + + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + methodCalls := body["methodCalls"].([]any) + call := methodCalls[0].([]any) + name := call[0].(string) + switch name { + case "x:Account/query": + ids := make([]string, len(accounts)) + for i, a := range accounts { + ids[i] = a["id"].(string) + } + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"x:Account/query", map[string]any{"ids": ids}, "q"}, + }}) + case "x:Account/get": + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"x:Account/get", map[string]any{"list": accounts}, "g"}, + }}) + case "Mailbox/get": + args := call[1].(map[string]any) + accountID := args["accountId"].(string) + target := strings.TrimPrefix(accountID, "mail-") + json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ + []any{"Mailbox/get", map[string]any{"list": mailboxesByEmail[target]}, "m"}, + }}) + } + })) + apiURL = srv.URL + "/api" + return srv +} + +func TestCompareContentIntegrityMatchesRewrittenBareUsernameByLocalPart(t *testing.T) { + // Pre-migration, the account was a bare username "alice" (pre-0.16 + // style). Post-migration, v0.16's own conversion rewrote it to a full + // email address - see UPGRADING/v0_16.md. An exact-string match would + // wrongly report "alice" as missing. + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "alice@example.com", "domainId": "example.com"}}, + map[string][]map[string]any{"alice@example.com": {{"name": "Inbox", "totalEmails": 42}}}, + ) + defer srv.Close() + + before := &checkpoint.PreflightSnapshot{ + MailboxCounts: map[string][]checkpoint.MailboxCount{ + "alice": {{Mailbox: "Inbox", Messages: 42}}, // bare username, pre-migration + }, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatalf("compareContentIntegrity: %v", err) + } + if !result.OK() { + t.Errorf("result.OK() = false, want true (local-part match should have found alice@example.com): %s", result.String()) + } + if len(result.MissingAccounts) != 0 { + t.Errorf("MissingAccounts = %v, want none", result.MissingAccounts) + } +} + +func TestCompareContentIntegrityNoFalseMatchAcrossUnrelatedAccounts(t *testing.T) { + // "alice" (before) must not spuriously match "alice-archive@example.com" + // (after) just because one contains the other - local-part comparison + // must be an exact match on the part before "@", not a substring check. + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "alice-archive@example.com", "domainId": "example.com"}}, + map[string][]map[string]any{"alice-archive@example.com": {{"name": "Inbox", "totalEmails": 1}}}, + ) + defer srv.Close() + + before := &checkpoint.PreflightSnapshot{ + MailboxCounts: map[string][]checkpoint.MailboxCount{ + "alice": {{Mailbox: "Inbox", Messages: 42}}, + }, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatalf("compareContentIntegrity: %v", err) + } + if result.OK() { + t.Fatal("result.OK() = true, want a missing-account failure - alice-archive@example.com is a different account than alice") + } + if len(result.MissingAccounts) != 1 || result.MissingAccounts[0] != "alice" { + t.Errorf("MissingAccounts = %v, want [alice]", result.MissingAccounts) + } +} + +func TestCompareContentIntegrityMultipleMailboxesPerAccount(t *testing.T) { + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "bob@example.org", "domainId": "example.org"}}, + map[string][]map[string]any{"bob@example.org": { + {"name": "Inbox", "totalEmails": 10}, + {"name": "Archive", "totalEmails": 200}, + }}, + ) + defer srv.Close() + + before := &checkpoint.PreflightSnapshot{ + MailboxCounts: map[string][]checkpoint.MailboxCount{ + "bob@example.org": { + {Mailbox: "Inbox", Messages: 10}, + {Mailbox: "Archive", Messages: 199}, // one message short + }, + }, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatalf("compareContentIntegrity: %v", err) + } + if result.AccountsChecked != 1 || result.MailboxesChecked != 2 { + t.Errorf("AccountsChecked=%d MailboxesChecked=%d, want 1 and 2", result.AccountsChecked, result.MailboxesChecked) + } + if len(result.MessageCountMismatches) != 1 { + t.Fatalf("MessageCountMismatches = %+v, want exactly one (Archive)", result.MessageCountMismatches) + } + m := result.MessageCountMismatches[0] + if m.Mailbox != "Archive" || m.Before != 199 || m.After != 200 { + t.Errorf("mismatch = %+v, want Archive 199->200", m) + } +} diff --git a/internal/validate/doc.go b/internal/validate/doc.go new file mode 100644 index 0000000..95255ab --- /dev/null +++ b/internal/validate/doc.go @@ -0,0 +1,3 @@ +// Package validate implements the post-migration validation suite and its structured report. +// See ARCHITECTURE.md §4.7 for the design. +package validate diff --git a/internal/validate/main_test.go b/internal/validate/main_test.go new file mode 100644 index 0000000..1a87cad --- /dev/null +++ b/internal/validate/main_test.go @@ -0,0 +1,101 @@ +package validate + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "testing" +) + +// TestMain lets this test binary also act as a fake Stalwart binary, +// mirroring internal/recovery's own TestMain - see that package's doc +// comment for why (the standard os/exec "helper process" technique). Beyond +// plain reachability, it also speaks just enough JMAP to serve +// stalwartapi.Client.AccountSnapshot (x:Account/query, x:Account/get, +// session discovery, Mailbox/get) for a single fixed fake account +// "alice@example.com", so BootCheck's content-integrity comparison can be +// exercised against a real subprocess rather than mocked in-process. The +// mailbox message count it reports is configurable via +// STALWART_MIGRATOR_TEST_MAILBOX_COUNT (default 42), so tests can produce +// both a matching and a mismatching post-migration snapshot. +func TestMain(m *testing.M) { + if os.Getenv("STALWART_MIGRATOR_TEST_HELPER") == "1" { + runFakeStalwartServer() + return + } + os.Exit(m.Run()) +} + +func runFakeStalwartServer() { + port := os.Getenv("STALWART_MIGRATOR_TEST_PORT") + messageCount := 42 + if v := os.Getenv("STALWART_MIGRATOR_TEST_MAILBOX_COUNT"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + messageCount = n + } + } + + ln, err := net.Listen("tcp", "127.0.0.1:"+port) + if err != nil { + fmt.Fprintln(os.Stderr, "fake stalwart: listen:", err) + os.Exit(1) + } + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap": + user, _, _ := r.BasicAuth() + if strings.Contains(user, "%") { + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": "http://127.0.0.1:" + port + "/api", + "primaryAccounts": map[string]string{"urn:ietf:params:jmap:mail": "mail-alice"}, + }) + return + } + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api": + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + methodCalls, _ := body["methodCalls"].([]any) + if len(methodCalls) == 0 { + w.WriteHeader(http.StatusBadRequest) + return + } + call := methodCalls[0].([]any) + name := call[0].(string) + switch name { + case "x:Account/query": + json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{ + []any{"x:Account/query", map[string]any{"ids": []string{"a1"}}, "q"}, + }}) + case "x:Account/get": + json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{ + []any{"x:Account/get", map[string]any{"list": []map[string]any{ + {"id": "a1", "name": "alice@example.com", "domainId": "example.com"}, + }}, "g"}, + }}) + case "Mailbox/get": + json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{ + []any{"Mailbox/get", map[string]any{"list": []map[string]any{ + {"name": "Inbox", "totalEmails": messageCount}, + }}, "m"}, + }}) + default: + w.WriteHeader(http.StatusBadRequest) + } + default: + w.WriteHeader(http.StatusOK) + } + })} + go srv.Serve(ln) + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM) + <-sigCh + os.Exit(0) +} diff --git a/internal/validate/report.go b/internal/validate/report.go new file mode 100644 index 0000000..a1d38f8 --- /dev/null +++ b/internal/validate/report.go @@ -0,0 +1,40 @@ +package validate + +import ( + "fmt" + "strings" +) + +type Status string + +const ( + StatusOK Status = "ok" + StatusFail Status = "fail" +) + +type CheckResult struct { + Name string + Status Status + Detail string +} + +type Report struct { + Results []CheckResult +} + +func (r Report) Blocking() bool { + for _, res := range r.Results { + if res.Status == StatusFail { + return true + } + } + return false +} + +func (r Report) String() string { + var b strings.Builder + for _, res := range r.Results { + fmt.Fprintf(&b, "[%-4s] %-16s %s\n", strings.ToUpper(string(res.Status)), res.Name, res.Detail) + } + return b.String() +}