diff --git a/docs/spec/cutover.md b/docs/spec/cutover.md index 724c12d..1ed0136 100644 --- a/docs/spec/cutover.md +++ b/docs/spec/cutover.md @@ -87,6 +87,15 @@ unprivileged process hold port 25. 3. Copy `/opt/stalwart/data` to the fork's store path. A copy, not a move: the original is the rollback. Budget the disk for two full copies, and take it from the stopped server, never from under a running one. + + **Then make the original read-only, before the fork exists on this host** + (`chmod -R a-w`, or `chattr +i` on the directory, or keep it on a + read-only bind mount). Until this moment the running server's own RocksDB + lock was what stopped anything else opening that store; stopping it takes + that away exactly when a store path is about to be typed. A read-only + original refuses the fork harmlessly and still starts under the + Enterprise build — measured, `tools/fork/cutover-rehearsal/probe_guard.py`. + Undo it only if you are rolling back. 4. Install the fork at its own path with its own config, pointing at the copied store. Most settings travel inside the store — they live in the registry — so the config file is mainly the store path and the hostname. @@ -234,6 +243,19 @@ database with `create_missing_column_families(true)`, so it creates `_` on first open. Upstream has no descriptor for it, and RocksDB will not open a database holding a column family it was not told about. +The mistake is worth guarding mechanically rather than carefully, because +the guard that exists today is removed by step 2. Three states, measured +(`probe_guard.py`): + +| The original store is | The fork | The rollback | +|---|---|---| +| held by the running server | refused by RocksDB's lock | intact | +| stopped, read-only | refused while rotating its own log | intact | +| stopped, writable | **opens, and adds `_`** | **gone** | + +So the window of exposure opens the moment `stalwart.service` stops and +closes when the original is made read-only. Keep it short. + Three consequences: - The failure is a hard one, at startup, exit code 1, **before any data is diff --git a/tools/fork/cutover-rehearsal/probe_guard.py b/tools/fork/cutover-rehearsal/probe_guard.py new file mode 100755 index 0000000..7a5c775 --- /dev/null +++ b/tools/fork/cutover-rehearsal/probe_guard.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +"""What actually stops the fork opening the original store by mistake. + +cutover.md step 3 says "a copy, not a move", and the rehearsal showed why: +one open by the fork adds its column family and the Enterprise build can +never start on that store again. So the question is what prevents the +mistake, mechanically, rather than by being careful at 1am. + +Three probes, in order, the last of which destroys the fixture: + + A the old server is running — does RocksDB's own lock refuse? + B stopped, store mounted read-only — does the fork refuse harmlessly? + C stopped, writable — the accident, confirmed end to end + +Run phase1.py first; this uses the install it leaves behind. +""" +import os, subprocess, sys, time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import lib + +OLD = lib.INSTALLS["old"]["dir"] +PROBE = "cutover-probe" + + +def try_fork_on(path, read_only=False, label=""): + """Point the fork at `path` and report how it fails, or that it didn't.""" + lib.docker("rm", "-f", PROBE, check_rc=False) + mount = f"{path}/data:/var/lib/stalwart" + (":ro" if read_only else "") + lib.docker( + "run", "-d", "--name", PROBE, + "--user", f"{os.getuid()}:{os.getgid()}", + "--entrypoint", "/usr/local/bin/inbuxa", + "-v", f"{lib.ROOT}/target/debug/inbuxa:/usr/local/bin/inbuxa:ro", + "-v", f"{path}/etc:/etc/stalwart" + (":ro" if read_only else ""), + "-v", mount, + lib.IMAGE, "--config", "/etc/stalwart/config.json", + check_rc=False, + ) + time.sleep(8) + state = lib.docker("inspect", "-f", "{{.State.Status}} exit={{.State.ExitCode}}", + PROBE, check_rc=False).stdout.strip() + log = lib.docker("logs", "--tail", "40", PROBE, check_rc=False) + text = (log.stdout or "") + (log.stderr or "") + lib.docker("rm", "-f", PROBE, check_rc=False) + first = next((l.strip() for l in text.splitlines() + if "failed" in l.lower() or "error" in l.lower()), "") + print(f" {label}: {state}") + if first: + print(f" {first[:170]}") + return state, text + + +def upstream_can_open(path): + """Whether upstream 0.16.22 still starts on this store — the rollback.""" + name = "cutover-upstream-check" + lib.docker("rm", "-f", name, check_rc=False) + lib.docker( + "run", "-d", "--name", name, + "--user", f"{os.getuid()}:{os.getgid()}", + "--entrypoint", "/usr/local/bin/stalwart", + "-v", f"{lib.ROOT}/target/debug/stalwart:/usr/local/bin/stalwart:ro", + "-v", f"{path}/etc:/etc/stalwart", + "-v", f"{path}/data:/var/lib/stalwart", + lib.IMAGE, "--config", "/etc/stalwart/config.json", + check_rc=False, + ) + time.sleep(10) + status = lib.docker("inspect", "-f", "{{.State.Status}}", name, check_rc=False).stdout.strip() + log = lib.docker("logs", "--tail", "20", name, check_rc=False) + text = (log.stdout or "") + (log.stderr or "") + lib.docker("rm", "-f", name, check_rc=False) + return status == "running", text + + +def main(): + print("== probe A: the old server is running\n") + if not lib.wait_http(5): + print(" the old install is not up — run phase1.py first") + sys.exit(2) + state, text = try_fork_on(OLD, label="fork pointed at the live store") + locked = "lock" in text.lower() or "LOCK" in text + lib.check(state.startswith("exited"), + "the fork refuses to open a store another process holds") + lib.check(locked, "it is RocksDB's lock that refuses it", + "so the running server is itself the guard") + + print("\n== probe B: stopped, but the store is read-only\n") + lib.stop("old") + time.sleep(2) + state, text = try_fork_on(OLD, read_only=True, label="fork pointed at a read-only store") + lib.check(state.startswith("exited"), "the fork refuses a read-only store") + ok, _ = upstream_can_open(OLD) + lib.check(ok, "upstream still starts on it — the rollback is intact", + "a read-only original survives being pointed at") + + print("\n== probe C: stopped and writable — the accident\n") + state, text = try_fork_on(OLD, label="fork pointed at the writable original") + ok, _ = upstream_can_open(OLD) + lib.check(not ok, "upstream can no longer start — the rollback is gone", + "one open was enough" if not ok else "unexpectedly survived") + + lib.summary("probe") + print("\nThe fixture is now poisoned by design; re-run phase1.py to rebuild it.") + + +if __name__ == "__main__": + main()