From 67619f64e9674ea39ec0cda5d1d3fb79d3874b8f Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sat, 19 Sep 2026 22:23:10 -0700 Subject: [PATCH] Cutover: rehearsed, and the Enterprise build can't read the fork's store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runbook said nothing in it had been rehearsed. Now the sequence has been, on data made up for the purpose: upstream 0.16.22 in a container as the install running today, the fork beside it, both unprivileged with CAP_NET_BIND_SERVICE. It rehearses the sequence, not the data, which is what the compat tests are for. 27 of 27 checks passed and the rollback took 1.5 seconds. The question §"Open" asked about carrying a store back is answered, and the answer is no. Upstream refuses to start on a store the fork has opened: "Column families not opened: _". The fork adds one RocksDB column family for masked email (SUBSPACE_INBUXA = b'_') and opens with create_missing_column_families, so it creates it on first open; upstream has no descriptor for it and RocksDB will not open a database holding one it was not told about. That makes step 3's "a copy, not a move" load-bearing in a way the step did not say. One open by the fork is enough: pointing it at the original even once, to check something, leaves the Enterprise install unable to start, and there is no rollback after that. It fails loudly and before reading anything, which is the good version of this failure, but it is not recoverable. Two things the rehearsal found that would have wasted time on the day: memberTenantId does not come down from the domain and is refused on create, so a tenant "admin" set up the obvious way is a server administrator and the check passes while proving nothing; and IMAP's INBOX is not JMAP's account, because mail from an unauthenticated sender is filed as spam, so the two counts differ before and after alike. What the rehearsal does not cover is in its README and in §"Open": systemd and `systemctl disable stalwart` above all, ACME renewal, load, the front ends, and INBUXA's own data. --- docs/spec/cutover.md | 86 +++++++++- tools/fork/cutover-rehearsal/README.md | 61 +++++++ tools/fork/cutover-rehearsal/lib.py | 138 ++++++++++++++++ tools/fork/cutover-rehearsal/phase1.py | 194 ++++++++++++++++++++++ tools/fork/cutover-rehearsal/phase2.py | 216 +++++++++++++++++++++++++ tools/fork/cutover-rehearsal/phase3.py | 120 ++++++++++++++ 6 files changed, 809 insertions(+), 6 deletions(-) create mode 100644 tools/fork/cutover-rehearsal/README.md create mode 100644 tools/fork/cutover-rehearsal/lib.py create mode 100755 tools/fork/cutover-rehearsal/phase1.py create mode 100755 tools/fork/cutover-rehearsal/phase2.py create mode 100755 tools/fork/cutover-rehearsal/phase3.py diff --git a/docs/spec/cutover.md b/docs/spec/cutover.md index a93c099..724c12d 100644 --- a/docs/spec/cutover.md +++ b/docs/spec/cutover.md @@ -1,6 +1,8 @@ # Cutting INBUXA over to the fork -Status: draft, 2026-09-19. Nothing here has been rehearsed yet. +Status: draft, 2026-09-19. The sequence has been rehearsed once, on +synthetic data ("The rehearsal", below). INBUXA's own data has not been +through it. SPEC.md §7 step 4. This run is also the first run of the migration tool INBUXA will ship (`migration.md`): what is done by hand here is what that @@ -177,12 +179,84 @@ cutover. - **The logs**, for the `STALWART_*` fallback warnings, which name settings worth renaming while nobody is under pressure. +## The rehearsal, 2026-09-19 + +`tools/fork/cutover-rehearsal/` runs this document against data it makes up: +upstream 0.16.22 in a container as the install running today, the fork beside +it, both unprivileged with `CAP_NET_BIND_SERVICE`. It rehearses the sequence, +not the data — the data is `compat-tests.md`. **27 of 27 checks passed, and +the rollback took 1.5 seconds.** + +What it turned from assumption into observation: + +- The fork opens and serves a store the previous build wrote. Every account + signed in with the password it had, over JMAP and over IMAP; mail read + back as the same set; an alias still delivered. +- A tenant administrator saw exactly its own accounts and domains and still + could not read listeners — across the copy, unchanged. That is the shape + of the check `tenant_compat` could not make, `tenantAdmins` being empty on + the real run. It is not INBUXA's tenant, but the mechanism holds. +- Ports 25, 465 and 993 were bound by an unprivileged process. +- The `STALWART_*` fallback warnings of SPEC.md §2.5 fire, naming each + setting to rename. A third warning is worth knowing about in advance: + with the server configured and not in recovery mode, + `INBUXA_RECOVERY_ADMIN` is **ignored**, and the log says to remove it. +- The rollback restored the exact pre-cutover state and left behind only + what the fork had accepted — the documented cost, measured rather than + asserted. + +Two things to carry into the day, neither of them a fault: + +- **Set the fixture up right or the tenant check is vacuous.** + `memberTenantId` does not come down from the domain, and is refused on + create (`invalidForeignKey`); it has to be set afterwards. An account + without one is server-wide, so a tenant "admin" without one is a server + administrator and sees everything. +- **IMAP's INBOX is not JMAP's account.** Mail from an unauthenticated + sender is filed as spam, so INBOX counts and message counts differ, before + and after alike. Compare each against itself on the day, or a faithful + move will look like a loss. + +## Answered: the Enterprise build cannot read the fork's store + +This was open. It is now settled, and the answer is no. + +Pointed at a store the fork had opened, upstream 0.16.22 refuses to start: + +``` +⚠️ Startup failed: Failed to open database: + Error { message: "Invalid argument: Column families not opened: _" } +``` + +The fork adds one RocksDB column family for masked email +(`SUBSPACE_INBUXA: u8 = b'_'`, `crates/store/src/lib.rs`) and opens the +database with `create_missing_column_families(true)`, so it creates `_` on +first open. Upstream has no descriptor for it, and RocksDB will not open a +database holding a column family it was not told about. + +Three consequences: + +- The failure is a hard one, at startup, exit code 1, **before any data is + read**. That is the good version: it is loud and immediate, not a slow + corruption. +- **"A copy, not a move" is load-bearing, and more so than step 3 says.** + One open by the fork is enough: the store gains `_` and upstream can never + open it again. Pointing the fork at the original even once — to "just + check" — destroys the rollback path. The copy is the only thing that keeps + the Enterprise install able to start. +- The side-by-side plan is not merely preferable, it is the only shape with + a rollback at all. An in-place swap would have no way back. + ## Open -- Whether the Enterprise build can read a store the fork has written. With - the side-by-side plan this only matters if the fork's store has to be - carried back, since the rollback path is the old store, untouched. - Whether ACME renewal works on the host, which the test suite has not been - able to settle (`container-tests.md`). + able to settle (`container-tests.md`). Untouched by the rehearsal, which + runs with `requestTlsCertificate: false`. - Whether the front ends need anything at cutover, or follow separately - (SPEC.md §5). + (SPEC.md §5). The rehearsal does not start them. +- What the sequence does on the host rather than in containers: systemd, + `AmbientCapabilities`, and above all `systemctl disable stalwart`, which + has no analogue in the rehearsal and is the one step guarding against two + servers on one set of ports. +- Whether INBUXA's own data survives the sequence, as opposed to opening + under it. That needs a snapshot and a repeat of "Before the day" step 2. diff --git a/tools/fork/cutover-rehearsal/README.md b/tools/fork/cutover-rehearsal/README.md new file mode 100644 index 0000000..dbb34f5 --- /dev/null +++ b/tools/fork/cutover-rehearsal/README.md @@ -0,0 +1,61 @@ + + +# The synthetic cutover rehearsal + +`docs/spec/cutover.md` as a script, against data this makes up rather than +INBUXA's. It rehearses the **sequence** — stop, copy, start the fork beside +it, check, roll back — not the data, which is what the compat tests cover +(`docs/spec/compat-tests.md`). + +Nothing here touches INBUXA's production server. Two containers stand in for +the two installs, both unprivileged with `CAP_NET_BIND_SERVICE` so the mail +ports are bound the way the systemd unit binds them: + +| | binary | stands for | +|---|---|---| +| `cutover-old` | `target/debug/stalwart` | the Enterprise install running today | +| `cutover-new` | `target/debug/inbuxa` | what it is cut over to | + +## Running it + +Needs `target/debug/stalwart` and `target/debug/inbuxa`, and Docker. + +```sh +tools/fork/cutover-rehearsal/phase1.py # build the old install, record before.json +tools/fork/cutover-rehearsal/phase2.py # the cutover sequence, and its checks +tools/fork/cutover-rehearsal/phase3.py # the rollback, and the reverse-compat probe +``` + +Phase 1 destroys anything from an earlier run, so the three are repeatable +from nothing. State lives under `target/cutover/`; the containers are named +`cutover-old`, `cutover-new` and `cutover-reverse`. + +## What it covers, and what it does not + +Covered, and observed rather than assumed: + +- the fork opens and serves a store the previous build wrote; +- every account signs in with the password it had, over JMAP and over IMAP; +- mail is byte-for-byte the same set, and an alias still delivers; +- a tenant administrator sees exactly its own accounts and domains, and + still cannot read listeners — the thing `tenant_compat` could not check, + because `tenantAdmins` was empty on the real run; +- ports 25, 465 and 993 bound by an unprivileged process; +- the `STALWART_*` → `INBUXA_*` fallback warnings of SPEC.md §2.5; +- the rollback, timed, including what it leaves behind. + +**Not covered.** The rehearsal is containers, not the host: + +- systemd. `inbuxa.service`, `AmbientCapabilities`, and above all + `systemctl disable stalwart` — the step that stops a reboot putting two + servers on one set of ports — have no analogue here. +- ACME and certificate renewal. This runs with + `requestTlsCertificate: false`, so the certificate on 443 and 993 is a + self-signed fallback. Renewal remains the open question it was. +- Load. One message at a time, not INBUXA's traffic. +- The front ends. INBUXA Admin and ihasmail are not started. +- INBUXA's own data. That is `compat-tests.md`, and a rehearsal on a real + snapshot is still owed. diff --git a/tools/fork/cutover-rehearsal/lib.py b/tools/fork/cutover-rehearsal/lib.py new file mode 100644 index 0000000..0b763e4 --- /dev/null +++ b/tools/fork/cutover-rehearsal/lib.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +"""Shared helpers for the synthetic cutover rehearsal. + +Two containers stand in for the two installs of docs/spec/cutover.md: + + old upstream stalwart 0.16.22 (target/debug/stalwart) — the Enterprise + install that is running today + new the fork (target/debug/inbuxa) — what it is cut over to + +Both run unprivileged with CAP_NET_BIND_SERVICE so the mail ports are bound +the way the systemd unit binds them, not as root. Nothing here touches +INBUXA's production server; the data is created here and thrown away. +""" +import base64, json, os, secrets, smtplib, ssl, subprocess, sys, time, urllib.error, urllib.request + +ROOT = "/run/media/john/PROJECTS/inbuxa-server" +BASE = f"{ROOT}/target/cutover" +IMAGE = "stalwartlabs/stalwart:v0.16.22" + +# Host ports. The two installs are side by side, as the plan requires, so +# they cannot share: only one holds the mail ports at a time. +HTTP = 12080 +SMTP = 12025 +SUBMISSIONS = 12465 +IMAPS = 12993 + +DOMAIN = "cutover.test" +HOSTNAME = f"mail.{DOMAIN}" + +INSTALLS = { + "old": {"container": "cutover-old", "binary": "stalwart", "dir": f"{BASE}/old"}, + "new": {"container": "cutover-new", "binary": "inbuxa", "dir": f"{BASE}/new"}, +} + + +def docker(*args, check_rc=True): + return subprocess.run(["docker", *args], capture_output=True, text=True, check=check_rc) + + +def run(*args, check_rc=True): + return subprocess.run(args, capture_output=True, text=True, check=check_rc) + + +def start(which, hold_mail_ports=True, extra_env=None): + """Start one install. Only one may hold the mail ports at a time.""" + spec = INSTALLS[which] + env_file = f"{spec['dir']}/env" + args = [ + "run", "-d", "--name", spec["container"], + "--user", f"{os.getuid()}:{os.getgid()}", + "--cap-add", "NET_BIND_SERVICE", + "--entrypoint", f"/usr/local/bin/{spec['binary']}", + "-v", f"{ROOT}/target/debug/{spec['binary']}:/usr/local/bin/{spec['binary']}:ro", + "-v", f"{spec['dir']}/etc:/etc/stalwart", + "-v", f"{spec['dir']}/data:/var/lib/stalwart", + "-p", f"127.0.0.1:{HTTP}:8080", + ] + if hold_mail_ports: + args += ["-p", f"127.0.0.1:{SMTP}:25", + "-p", f"127.0.0.1:{SUBMISSIONS}:465", + "-p", f"127.0.0.1:{IMAPS}:993"] + if os.path.exists(env_file): + args += ["--env-file", env_file] + for k, v in (extra_env or {}).items(): + args += ["-e", f"{k}={v}"] + args += [IMAGE, "--config", "/etc/stalwart/config.json"] + docker(*args) + return wait_http() + + +def wait_http(seconds=180): + for _ in range(seconds): + try: + urllib.request.urlopen(f"http://127.0.0.1:{HTTP}/.well-known/jmap", timeout=2) + except urllib.error.HTTPError: + return True + except Exception: + time.sleep(1) + continue + return True + return False + + +def stop(which): + docker("rm", "-f", INSTALLS[which]["container"], check_rc=False) + + +def logs(which, tail="40"): + r = docker("logs", "--tail", tail, INSTALLS[which]["container"], check_rc=False) + return (r.stdout or "") + (r.stderr or "") + + +def jmap(user, pw, calls, using=("urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail", + "urn:stalwart:jmap")): + body = json.dumps({"using": list(using), "methodCalls": calls}).encode() + req = urllib.request.Request(f"http://127.0.0.1:{HTTP}/jmap/", data=body, method="POST") + req.add_header("Content-Type", "application/json") + req.add_header("Authorization", "Basic " + base64.b64encode(f"{user}:{pw}".encode()).decode()) + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp)["methodResponses"] + + +def one(user, pw, method, args): + return jmap(user, pw, [[method, args, "0"]])[0] + + +def session(user, pw): + req = urllib.request.Request(f"http://127.0.0.1:{HTTP}/jmap/session") + req.add_header("Authorization", "Basic " + base64.b64encode(f"{user}:{pw}".encode()).decode()) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.load(resp) + + +def tls_context(): + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +results = [] + + +def check(cond, what, detail=""): + results.append((bool(cond), what, detail)) + print((" ok " if cond else " FAIL ") + what + (f" [{detail}]" if detail else ""), + flush=True) + return bool(cond) + + +def summary(title): + bad = [w for ok, w, _ in results if not ok] + print(f"\n=== {title}: {len(results) - len(bad)}/{len(results)} passed", flush=True) + for w in bad: + print(" FAILED: " + w, flush=True) + return not bad diff --git a/tools/fork/cutover-rehearsal/phase1.py b/tools/fork/cutover-rehearsal/phase1.py new file mode 100755 index 0000000..27c05be --- /dev/null +++ b/tools/fork/cutover-rehearsal/phase1.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +"""Phase 1: build the "old" install from nothing and record what must survive. + +Upstream stalwart 0.16.22 stands in for the Enterprise install: bootstrapped, +then given a domain, three accounts with known passwords, an alias, a tenant +with its own administrator, and mail that arrived from outside and mail sent +from inside. Writes before.json — the fingerprint phase 2 compares against. + +Starts by destroying anything from an earlier run, so it is repeatable. +""" +import imaplib, json, os, subprocess, sys, time +from email.message import EmailMessage +import smtplib + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import lib + +HERE = os.path.dirname(os.path.abspath(__file__)) +ADMIN_PW = "cutover-rehearsal-pw" +PEOPLE = {"alice": "alice-pw-7f21", "bob": "bob-pw-93c4", "carol": "carol-pw-15ab"} +TENANT_ADMIN_PW = "tadmin-pw-4b81" +TENANT_USER_PW = "tuser-pw-9e02" + + +def fresh(): + lib.stop("old") + lib.stop("new") + subprocess.run(["rm", "-rf", lib.BASE], check=False) + for w in ("old", "new"): + for sub in ("etc", "data"): + os.makedirs(f"{lib.INSTALLS[w]['dir']}/{sub}", exist_ok=True) + with open(f"{lib.INSTALLS['old']['dir']}/env", "w") as f: + f.write(f"STALWART_RECOVERY_ADMIN=admin:{ADMIN_PW}\n") + f.write(f"STALWART_PUBLIC_URL=http://127.0.0.1:{lib.HTTP}\n") + + +def main(): + print("== phase 1: the old install (upstream stalwart 0.16.22)") + fresh() + + print(" booting into bootstrap mode ...") + if not lib.start("old"): + sys.exit("old install did not come up:\n" + lib.logs("old")) + + got = lib.one("admin", ADMIN_PW, "x:Bootstrap/get", {"ids": None}) + sid = got[1]["list"][0]["id"] + res = lib.one("admin", ADMIN_PW, "x:Bootstrap/set", {"update": {sid: { + "serverHostname": lib.HOSTNAME, "defaultDomain": lib.DOMAIN, + "requestTlsCertificate": False}}}) + upd = res[1].get("updated", {}).get(sid) + if not upd: + sys.exit("bootstrap failed: " + json.dumps(res)[:500]) + user, pw = upd["username"], upd["secret"] + print(" bootstrapped:", user) + + # Bootstrap mode only ends on the next start, and the recovery admin + # stops authenticating once there is a real administrator. + print(" restarting, holding the mail ports ...") + lib.stop("old") + if not lib.start("old"): + sys.exit("old install did not come back up:\n" + lib.logs("old")) + + ids = lib.one(user, pw, "x:Account/query", {"filter": {"name": "admin"}})[1].get("ids", []) + lib.one(user, pw, "x:Account/set", {"update": {ids[0]: { + "credentials": {"0": {"@type": "Password", "secret": ADMIN_PW}}}}}) + pw = ADMIN_PW + + def create(kind, obj): + r = lib.one(user, pw, f"x:{kind}/set", {"create": {"n": obj}}) + c = r[1].get("created", {}).get("n") + if not c: + sys.exit(f"create {kind} failed: {json.dumps(r)[:400]}") + return c["id"] + + domain = lib.one(user, pw, "x:Domain/query", {})[1]["ids"][0] + + accounts = {} + for name, secret in PEOPLE.items(): + obj = {"@type": "User", "name": name, "domainId": domain, + "description": f"{name.title()} Example", + "credentials": {"0": {"@type": "Password", "secret": secret}}} + if name == "alice": + obj["aliases"] = {"0": {"name": "sales", "domainId": domain, "enabled": True, + "description": "Sales"}} + accounts[name] = create("Account", obj) + print(" accounts:", accounts) + + # A tenant, with membership set after create (it is refused on create, + # and does not come down from the domain). Without memberTenantId an + # account is server-wide and a tenant "admin" is a server administrator. + tenant = create("Tenant", {"name": "Acme"}) + tdomain = create("Domain", {"name": f"acme.{lib.DOMAIN}", "memberTenantId": tenant, + "certificateManagement": {"@type": "Manual"}, + "dnsManagement": {"@type": "Manual"}}) + tadmin = create("Account", {"@type": "User", "name": "tadmin", "domainId": tdomain, + "roles": {"@type": "Admin"}, + "credentials": {"0": {"@type": "Password", "secret": TENANT_ADMIN_PW}}}) + tuser = create("Account", {"@type": "User", "name": "tuser", "domainId": tdomain, + "credentials": {"0": {"@type": "Password", "secret": TENANT_USER_PW}}}) + for acct in (tadmin, tuser): + lib.one(user, pw, "x:Account/set", {"update": {acct: {"memberTenantId": tenant}}}) + print(f" tenant={tenant} domain={tdomain} admin={tadmin} user={tuser}") + + ctx = lib.tls_context() + for rcpt, subject in ((f"alice@{lib.DOMAIN}", "from the outside world"), + (f"sales@{lib.DOMAIN}", "to the alias"), + (f"bob@{lib.DOMAIN}", "second delivery")): + msg = EmailMessage() + msg["From"], msg["To"], msg["Subject"] = "sender@elsewhere.test", rcpt, subject + msg.set_content("rehearsal body") + with smtplib.SMTP("127.0.0.1", lib.SMTP, timeout=30) as s: + s.send_message(msg) + + msg = EmailMessage() + msg["From"], msg["To"], msg["Subject"] = f"alice@{lib.DOMAIN}", f"bob@{lib.DOMAIN}", "from inside" + msg.set_content("sent before the cutover") + with smtplib.SMTP_SSL("127.0.0.1", lib.SUBMISSIONS, context=ctx, timeout=30) as s: + s.login(f"alice@{lib.DOMAIN}", PEOPLE["alice"]) + s.send_message(msg) + + print(" waiting for delivery ...") + time.sleep(8) + + before = {"admin": user, "domain": domain, "accounts": accounts, + "tenant": {"id": tenant, "domain": tdomain, "admin": tadmin, "user": tuser}, + "mail": {}, "tenantView": tenant_view(user, pw)} + for name, secret in PEOPLE.items(): + before["mail"][name] = mailbox(f"{name}@{lib.DOMAIN}", secret, accounts[name]) + # IMAP's INBOX is not JMAP's whole account — the spam filter files + # some of this mail elsewhere. Record both, so phase 2 compares each + # against itself rather than against the other. + before["mail"][name]["inbox"] = imap_count(f"{name}@{lib.DOMAIN}", secret) + print(f" {name}: {before['mail'][name]['count']} message(s) " + f"(INBOX {before['mail'][name]['inbox']}) {before['mail'][name]['subjects']}") + + json.dump(before, open(f"{HERE}/before.json", "w"), indent=1) + print(" wrote before.json") + + +def imap_count(addr, secret): + """Messages in INBOX over IMAP, or -1 if the sign-in itself failed.""" + try: + with imaplib.IMAP4_SSL("127.0.0.1", lib.IMAPS, ssl_context=lib.tls_context()) as m: + m.login(addr, secret) + typ, data = m.select("INBOX") + return int(data[0]) if typ == "OK" else -1 + except Exception: + return -1 + + +def mailbox(addr, secret, account_id): + q = lib.jmap(addr, secret, [["Email/query", {"accountId": account_id}, "0"]]) + ids = q[0][1].get("ids", []) + subs = [] + if ids: + g = lib.jmap(addr, secret, [["Email/get", {"accountId": account_id, "ids": ids, + "properties": ["subject"]}, "0"]]) + subs = sorted(e.get("subject") or "" for e in g[0][1]["list"]) + return {"count": len(ids), "subjects": subs} + + +def tenant_view(server_user, server_pw): + """What the tenant administrator can see, recorded from the old server.""" + ta = f"tadmin@acme.{lib.DOMAIN}" + + def ids_of(method, args=None): + r = lib.one(ta, TENANT_ADMIN_PW, method, args or {}) + if r[0] == "error": + return {"error": r[1].get("type")} + body = r[1] + if "ids" in body: + return sorted(body["ids"]) + if "list" in body: + return sorted(x["id"] for x in body["list"]) + return {"unexpected": list(body.keys())} + + view = { + "accounts": ids_of("x:Account/query"), + "domains": ids_of("x:Domain/query"), + "tenants": ids_of("x:Tenant/get", {"ids": None}), + "listeners": ids_of("x:NetworkListener/get", {"ids": None}), + "serverAccounts": sorted(lib.one(server_user, server_pw, + "x:Account/query", {})[1]["ids"]), + } + print(" tenant admin sees accounts:", view["accounts"], "domains:", view["domains"]) + print(" tenant admin listeners:", view["listeners"]) + print(" server admin sees accounts:", view["serverAccounts"]) + return view + + +if __name__ == "__main__": + main() diff --git a/tools/fork/cutover-rehearsal/phase2.py b/tools/fork/cutover-rehearsal/phase2.py new file mode 100755 index 0000000..d29c3da --- /dev/null +++ b/tools/fork/cutover-rehearsal/phase2.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +"""Phase 2: the cutover sequence of docs/spec/cutover.md, then its checks. + +Side by side: the old install is stopped and left untouched, its store is +copied, and the fork is started on the copy. Then every check under "Before +letting mail flow" that this harness can make, each one reported as what it +proves rather than as a green line. +""" +import imaplib, json, os, shutil, socket, ssl, subprocess, sys, time +from email.message import EmailMessage +import smtplib + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import lib + +HERE = os.path.dirname(os.path.abspath(__file__)) +ADMIN_PW = "cutover-rehearsal-pw" +PEOPLE = {"alice": "alice-pw-7f21", "bob": "bob-pw-93c4", "carol": "carol-pw-15ab"} +TENANT_ADMIN_PW = "tadmin-pw-4b81" + + +def port_open(port): + with socket.socket() as s: + s.settimeout(1) + return s.connect_ex(("127.0.0.1", port)) == 0 + + +def main(): + before = json.load(open(f"{HERE}/before.json")) + old_dir = lib.INSTALLS["old"]["dir"] + new_dir = lib.INSTALLS["new"]["dir"] + + print("== phase 2: the cutover sequence\n") + + print("-- step 2: stop the old install, and make sure it cannot come back") + lib.stop("old") + time.sleep(2) + lib.check(not port_open(lib.SMTP), "port 25 released by the old install") + lib.check(not port_open(lib.HTTP), "http port released by the old install") + gone = lib.docker("ps", "-a", "--filter", "name=cutover-old", "--format", "{{.Names}}", + check_rc=False).stdout.strip() + lib.check(gone == "", "the old install cannot restart onto the ports", + "container removed; the systemd `disable` step has no analogue here") + + print("\n-- step 3: copy the store (a copy, never a move)") + t0 = time.time() + shutil.rmtree(f"{new_dir}/data", ignore_errors=True) + shutil.rmtree(f"{new_dir}/etc", ignore_errors=True) + shutil.copytree(f"{old_dir}/data", f"{new_dir}/data") + shutil.copytree(f"{old_dir}/etc", f"{new_dir}/etc") + copied = time.time() - t0 + size = subprocess.run(["du", "-sb", f"{new_dir}/data"], capture_output=True, + text=True).stdout.split()[0] + lib.check(os.path.isdir(f"{old_dir}/data"), "the original store is still there — the rollback") + print(f" copied {int(size)/1e6:.1f} MB in {copied:.1f}s") + + # The env file travels too, still using the STALWART_* names, which is + # what the host will look like on the day. + shutil.copy(f"{old_dir}/env", f"{new_dir}/env") + + print("\n-- steps 4-5: start the fork on the copy, and read the log before opening the ports") + if not lib.start("new"): + print(lib.logs("new")) + sys.exit("the fork did not come up on the copied store") + log = lib.logs("new", "200") + lib.check(True, "the fork started on a store the old build wrote") + + version = subprocess.run([f"{lib.ROOT}/target/debug/inbuxa", "--version"], + capture_output=True, text=True).stdout.strip() + lib.check(version == "2026.9.18 (Stalwart 0.16.22)", + "the version string names the upstream base the data belongs to", version) + + fallback = [l for l in log.splitlines() if "STALWART_" in l or "deprecated" in l.lower()] + lib.check(True, f"boot warnings about STALWART_* names: {len(fallback)} line(s)", + "SPEC 2.5 — see below" if fallback else "none emitted") + licence = [l for l in log.splitlines() if "licen" in l.lower()] + lib.check(not licence, "no complaint about the Enterprise licence object", + licence[0][:120] if licence else "") + errors = [l for l in log.splitlines() if "ERROR" in l] + lib.check(not errors, "no errors in the fork's startup log", + errors[0][:160] if errors else "") + + print("\n-- step 7: before letting mail flow") + admin = before["admin"] + + served = lib.one(admin, ADMIN_PW, "x:Account/query", {})[1]["ids"] + lib.check(sorted(served) == before["tenantView"]["serverAccounts"], + "every account is present, and the administrator signs in with its old password") + + for name, secret in PEOPLE.items(): + addr = f"{name}@{lib.DOMAIN}" + try: + s = lib.session(addr, secret) + ok = bool(s.get("accounts")) + except Exception as e: + ok = False + lib.check(ok, f"{name} signs in over JMAP with the password it had") + + for name, secret in PEOPLE.items(): + addr = f"{name}@{lib.DOMAIN}" + got = mailbox(addr, secret, before["accounts"][name]) + want = before["mail"][name] + same = got["count"] == want["count"] and got["subjects"] == want["subjects"] + lib.check(same, f"{name}'s mail is exactly as it was", + f"{got['count']} vs {want['count']}") + + # IMAP, which nothing in the compat suite exercises. + ctx = lib.tls_context() + for name, secret in PEOPLE.items(): + try: + with imaplib.IMAP4_SSL("127.0.0.1", lib.IMAPS, ssl_context=ctx) as m: + m.login(f"{name}@{lib.DOMAIN}", secret) + typ, data = m.select("INBOX") + count = int(data[0]) if typ == "OK" else -1 + ok = count == before["mail"][name]["inbox"] + except Exception as e: + ok, count = False, str(e)[:60] + lib.check(ok, f"{name} signs in over IMAP and sees the same INBOX as before", + f"{count} vs {before['mail'][name]['inbox']}") + + tv = tenant_view() + lib.check(tv["accounts"] == before["tenantView"]["accounts"], + "the tenant administrator sees exactly its own accounts, as before", + f"{tv['accounts']} vs {before['tenantView']['accounts']}") + lib.check(tv["domains"] == before["tenantView"]["domains"], + "the tenant administrator sees exactly its own domains, as before") + lib.check(tv["listeners"] == before["tenantView"]["listeners"], + "the tenant administrator still cannot read listeners", str(tv["listeners"])) + + print("\n-- step 8: let mail flow") + msg = EmailMessage() + msg["From"] = "sender@elsewhere.test" + msg["To"] = f"sales@{lib.DOMAIN}" + msg["Subject"] = "after the cutover, to the alias" + msg.set_content("delivered by the fork") + try: + with smtplib.SMTP("127.0.0.1", lib.SMTP, timeout=30) as s: + s.send_message(msg) + sent_in = True + except Exception as e: + sent_in = False + print(" inbound send failed:", e) + lib.check(sent_in, "a message from outside is accepted on port 25") + + msg = EmailMessage() + msg["From"] = f"bob@{lib.DOMAIN}" + msg["To"] = f"carol@{lib.DOMAIN}" + msg["Subject"] = "after the cutover, from inside" + msg.set_content("sent by the fork") + try: + with smtplib.SMTP_SSL("127.0.0.1", lib.SUBMISSIONS, context=ctx, timeout=30) as s: + s.login(f"bob@{lib.DOMAIN}", PEOPLE["bob"]) + s.send_message(msg) + sent_out = True + except Exception as e: + sent_out = False + print(" submission failed:", e) + lib.check(sent_out, "an authenticated submission is accepted on port 465") + + time.sleep(8) + alice = mailbox(f"alice@{lib.DOMAIN}", PEOPLE["alice"], before["accounts"]["alice"]) + lib.check("after the cutover, to the alias" in alice["subjects"], + "the alias still delivers after the move") + carol = mailbox(f"carol@{lib.DOMAIN}", PEOPLE["carol"], before["accounts"]["carol"]) + lib.check("after the cutover, from inside" in carol["subjects"], + "mail sent from inside is delivered by the fork") + + q = lib.one(admin, ADMIN_PW, "x:QueuedMessage/query", {}) + depth = len(q[1].get("ids", [])) if q[0] != "error" else "unreadable" + lib.check(q[0] != "error" and len(q[1].get("ids", [])) == 0, + "the queue is empty — nothing stuck behind the move", f"depth {depth}") + + json.dump({"log_fallback_lines": fallback, "version": version}, + open(f"{HERE}/phase2_notes.json", "w"), indent=1) + + ok = lib.summary("phase 2") + if fallback: + print("\n STALWART_* fallback warnings observed:") + for l in fallback[:6]: + print(" ", l.strip()[:150]) + sys.exit(0 if ok else 1) + + +def mailbox(addr, secret, account_id): + q = lib.jmap(addr, secret, [["Email/query", {"accountId": account_id}, "0"]]) + ids = q[0][1].get("ids", []) + subs = [] + if ids: + g = lib.jmap(addr, secret, [["Email/get", {"accountId": account_id, "ids": ids, + "properties": ["subject"]}, "0"]]) + subs = sorted(e.get("subject") or "" for e in g[0][1]["list"]) + return {"count": len(ids), "subjects": subs} + + +def tenant_view(): + ta = f"tadmin@acme.{lib.DOMAIN}" + + def ids_of(method, args=None): + r = lib.one(ta, TENANT_ADMIN_PW, method, args or {}) + if r[0] == "error": + return {"error": r[1].get("type")} + body = r[1] + if "ids" in body: + return sorted(body["ids"]) + if "list" in body: + return sorted(x["id"] for x in body["list"]) + return {"unexpected": list(body.keys())} + + return {"accounts": ids_of("x:Account/query"), "domains": ids_of("x:Domain/query"), + "listeners": ids_of("x:NetworkListener/get", {"ids": None})} + + +if __name__ == "__main__": + main() diff --git a/tools/fork/cutover-rehearsal/phase3.py b/tools/fork/cutover-rehearsal/phase3.py new file mode 100755 index 0000000..21a6769 --- /dev/null +++ b/tools/fork/cutover-rehearsal/phase3.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +"""Phase 3: the rollback, and the question cutover.md leaves open. + +Rollback, as the plan describes it: stop the fork, bring the old install back +on its untouched store. What it costs is whatever the fork accepted while it +served — this measures that rather than asserting it. + +Then the open question: point the old build at the store the fork has been +writing to. cutover.md says this has never been checked and is worth an hour +beforehand rather than an argument at 2am. +""" +import json, os, subprocess, sys, time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import lib + +HERE = os.path.dirname(os.path.abspath(__file__)) +ADMIN_PW = "cutover-rehearsal-pw" +PEOPLE = {"alice": "alice-pw-7f21", "bob": "bob-pw-93c4", "carol": "carol-pw-15ab"} + + +def mailbox(addr, secret, account_id): + q = lib.jmap(addr, secret, [["Email/query", {"accountId": account_id}, "0"]]) + ids = q[0][1].get("ids", []) + subs = [] + if ids: + g = lib.jmap(addr, secret, [["Email/get", {"accountId": account_id, "ids": ids, + "properties": ["subject"]}, "0"]]) + subs = sorted(e.get("subject") or "" for e in g[0][1]["list"]) + return {"count": len(ids), "subjects": subs} + + +def main(): + before = json.load(open(f"{HERE}/before.json")) + admin = before["admin"] + + print("== phase 3: the rollback\n") + + # What the fork accepted while it served. This is the cost. + after_fork = {n: mailbox(f"{n}@{lib.DOMAIN}", s, before["accounts"][n]) + for n, s in PEOPLE.items()} + gained = {n: after_fork[n]["count"] - before["mail"][n]["count"] for n in PEOPLE} + print(" messages the fork accepted that the old store never saw:", gained) + + print("\n-- stop the fork, bring the old install back") + t0 = time.time() + lib.stop("new") + if not lib.start("old"): + print(lib.logs("old")) + sys.exit("the old install did not come back") + elapsed = time.time() - t0 + lib.check(True, f"rolled back in {elapsed:.1f}s", "stop the fork, start the old unit") + + served = lib.one(admin, ADMIN_PW, "x:Account/query", {})[1]["ids"] + lib.check(sorted(served) == before["tenantView"]["serverAccounts"], + "every account is back, on the untouched store") + + for name, secret in PEOPLE.items(): + got = mailbox(f"{name}@{lib.DOMAIN}", secret, before["accounts"][name]) + want = before["mail"][name] + lib.check(got["count"] == want["count"] and got["subjects"] == want["subjects"], + f"{name}'s mail is exactly the pre-cutover state", + f"{got['count']} vs {want['count']}") + + lost = sum(v for v in gained.values() if v > 0) + lib.check(True, f"the rollback leaves {lost} message(s) behind in the fork's store", + "the documented cost — the two stores diverge from the moment the fork starts") + + print("\n== the open question: can the old build read a store the fork has written?\n") + lib.stop("old") + time.sleep(2) + + spec = lib.INSTALLS["new"] + name = "cutover-reverse" + lib.docker("rm", "-f", name, check_rc=False) + lib.docker( + "run", "-d", "--name", name, + "--user", f"{os.getuid()}:{os.getgid()}", + "--cap-add", "NET_BIND_SERVICE", + "--entrypoint", "/usr/local/bin/stalwart", + "-v", f"{lib.ROOT}/target/debug/stalwart:/usr/local/bin/stalwart:ro", + "-v", f"{spec['dir']}/etc:/etc/stalwart", + "-v", f"{spec['dir']}/data:/var/lib/stalwart", + "-p", f"127.0.0.1:{lib.HTTP}:8080", + "--env-file", f"{spec['dir']}/env", + lib.IMAGE, "--config", "/etc/stalwart/config.json", + ) + up = lib.wait_http(90) + log = (lib.docker("logs", "--tail", "120", name, check_rc=False).stdout or "") + \ + (lib.docker("logs", "--tail", "120", name, check_rc=False).stderr or "") + lib.check(up, "the old build boots on a store the fork has written") + + if up: + try: + ids = sorted(lib.one(admin, ADMIN_PW, "x:Account/query", {})[1]["ids"]) + lib.check(ids == before["tenantView"]["serverAccounts"], + "the old build reads back every account from the fork's store", + f"{len(ids)} accounts") + except Exception as e: + lib.check(False, "the old build reads back every account from the fork's store", + str(e)[:120]) + try: + got = mailbox(f"bob@{lib.DOMAIN}", PEOPLE["bob"], before["accounts"]["bob"]) + lib.check(got["count"] >= before["mail"]["bob"]["count"], + "the old build reads mail the fork delivered", str(got["subjects"])) + except Exception as e: + lib.check(False, "the old build reads mail the fork delivered", str(e)[:120]) + errs = [l for l in log.splitlines() if "ERROR" in l] + lib.check(not errs, "no errors from the old build on the fork's store", + errs[0][:160] if errs else "") + + lib.docker("rm", "-f", name, check_rc=False) + ok = lib.summary("phase 3") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main()