Implement internal/rollback and the service control it needs

Rollback was the one phase gating everything else: `run` without --dry-run
refused because this tool could not undo a cutover it had committed to.
That reason is now gone, and the refusal has narrowed to the fact that
there is no real cutover to undo yet.

internal/rollback implements ARCHITECTURE.md 4.8 as eight checkpointed
steps under PhaseRollback: verify-backup, stop-service,
preserve-failed-state, restore-data, restore-binary,
restore-service-config, start-service, verify-rollback.

Three things depart from what 4.8 specified, each for a reason:

- The backup is re-verified against its manifest *before* the service is
  stopped, which the design didn't call out. Finding a corrupt backup is
  survivable while the failed instance is still up, and unsurvivable once
  its data directory has been moved aside.
- BuildPlan is separate from Run, so every reason to refuse (closed
  rollback window, FoundationDB, no recorded backup, unknown deployment
  kind, missing database credentials) is found before anything is touched.
  The CLI prints that resolved plan and acts only with --yes.
- The restore is re-verified against the same manifest after writing. A
  restore that put back truncated bytes and reported success would be
  worse than one that failed outright.

Nothing from the failed attempt is deleted: the half-migrated data
directory and the displaced binary are moved to .failed-<run-id> names, so
a retry after the underlying issue is fixed still has both the evidence and
the artifacts. Afterwards a reduced validation suite runs against the
*restored* instance (version, reachability, directory counts) rather than
assuming the restore worked.

internal/service is a new package holding the systemd/Docker control this
needs. It's separate rather than living inside internal/rollback because
cutover will need the identical operations, and because the commands that
can take mail delivery down belong in one auditable place - the same
reasoning that makes stalwartapi the only thing speaking JMAP.
preflight.DeploymentKind is now a type alias for service.Kind so detection
and control can't drift apart. Its Active() reads `systemctl is-active`'s
output rather than its exit status: systemctl exits non-zero for every
non-active state, so exit-status logic would make "inactive" - the answer a
rollback most needs - look like a failure to read the state at all.

Also fixes a pre-existing bug in `status`: Go's flag package stops parsing
at the first positional argument, so `status <run-id> --state-dir X` looked
the run up in the default directory and reported it missing. `rollback`
would have inherited the same footgun on a command whose flags decide what
gets overwritten.

Still open: `confirm` cannot set RollbackWindowClosed. Rollback honours the
flag and refuses when it's set, but closing the window is the point of no
return for the backups this restores from, so it should land with the
retention policy 6 describes rather than before it.

Verified end to end against a fake systemd deployment: half-migrated data
restored to its original contents, failed state preserved, old binary
reinstalled and reporting 0.15.5, unit restarted, and a re-run of the
completed rollback inert.

cmd/stalwart-migrate/main.go

@@ -24,7 +24,7 @@ func main() {
 	case "status":
 		err = runStatus(os.Args[2:])
 	case "rollback":
-		err = fmt.Errorf("not implemented yet: see internal/rollback")
+		err = runRollback(os.Args[2:])
 	case "confirm":
 		err = fmt.Errorf("not implemented yet: see internal/checkpoint")
 	case "report":
@@ -47,7 +47,7 @@ commands:
   preflight   run read-only checks and print the migration plan
   run         --dry-run: simulate and validate against a sandbox (real cutover isn't implemented yet)
   status      show the state of an in-progress or completed run
-  rollback    restore the pre-migration backup for a given run
+  rollback    restore the pre-migration backup for a given run (prints the plan; --yes to act)
   confirm     close the rollback window for a completed run
   report      print the validation report for a run`)
 }

cmd/stalwart-migrate/rollback.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/backup"
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/rollback"
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/service"
+)
+
+// splitRunID pulls the run-id out of args wherever it appears, so
+// `rollback <run-id> --data-dir X` works as naturally as
+// `rollback --data-dir X <run-id>`. Go's flag package stops parsing at the
+// first positional argument, which would otherwise make the obvious
+// invocation order silently drop every flag after the run-id - on a command
+// whose flags decide what gets overwritten, that is not a failure mode
+// worth living with. Tokens consumed as a flag's value are skipped by
+// asking the FlagSet itself which flags take one, rather than by
+// maintaining a second list of them here.
+func splitRunID(fs *flag.FlagSet, args []string) (runID string, rest []string) {
+	rest = make([]string, 0, len(args))
+	for i := 0; i < len(args); i++ {
+		arg := args[i]
+		if arg == "--" {
+			rest = append(rest, args[i:]...)
+			break
+		}
+		if strings.HasPrefix(arg, "-") {
+			rest = append(rest, arg)
+			name := strings.TrimLeft(arg, "-")
+			if !strings.Contains(arg, "=") && takesValue(fs, name) && i+1 < len(args) {
+				i++
+				rest = append(rest, args[i])
+			}
+			continue
+		}
+		if runID == "" {
+			runID = arg
+			continue
+		}
+		rest = append(rest, arg) // a second positional: let Parse report it
+	}
+	return runID, rest
+}
+
+func takesValue(fs *flag.FlagSet, name string) bool {
+	f := fs.Lookup(name)
+	if f == nil {
+		return false
+	}
+	boolFlag, ok := f.Value.(interface{ IsBoolFlag() bool })
+	return !ok || !boolFlag.IsBoolFlag()
+}
+
+// runRollback implements `stalwart-migrate rollback <run-id>`: put the
+// instance back the way it was before the named run touched it
+// (ARCHITECTURE.md §4.8).
+//
+// This is the only command that stops a running mail server and overwrites
+// a live data directory, so it always prints the resolved plan first and
+// refuses to act without --yes. Everything the plan needs comes from the
+// run's own checkpoint - which backup, which manifest, which preserved
+// binary - so an operator rolling back days after the fact doesn't have to
+// remember any of it. The flags below exist to supply what the checkpoint
+// deliberately doesn't store (database credentials) or to override a
+// detection that was wrong.
+func runRollback(args []string) error {
+	fs := flag.NewFlagSet("rollback", flag.ExitOnError)
+	stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory runs are checkpointed in")
+	dataDir := fs.String("data-dir", "/var/lib/stalwart", "stalwart data directory to restore into (embedded backends)")
+	binaryPath := fs.String("binary", "/usr/local/bin/stalwart", "path the preserved old binary is reinstalled to")
+	serviceUnitPath := fs.String("service-unit", "", "path a preserved systemd unit or Compose file is restored to (only used if the run recorded one)")
+
+	deployment := fs.String("deployment", "", `override how the service is controlled: "systemd" or "docker" (default: whatever preflight detected for this run)`)
+	unitName := fs.String("unit", "stalwart", "systemd unit name")
+	containerName := fs.String("container", "stalwart", "docker container name")
+	stopTimeout := fs.Duration("stop-timeout", 60*time.Second, "how long to wait for the service to actually stop")
+	startTimeout := fs.Duration("start-timeout", 60*time.Second, "how long to wait for the restored service to come back")
+
+	adminURL := fs.String("admin-url", "", "base URL for the restored instance's admin/JMAP API, for post-rollback verification")
+	adminUser := fs.String("admin-user", "", "admin username")
+	adminPassword := fs.String("admin-password", os.Getenv("STALWART_MIGRATE_ADMIN_PASSWORD"),
+		"admin password (or set STALWART_MIGRATE_ADMIN_PASSWORD)")
+	verifyTimeout := fs.Duration("verify-timeout", 60*time.Second, "how long to wait for the restored instance to answer")
+
+	dbHost := fs.String("db-host", "", "external database host (postgresql/mysql backends)")
+	dbPort := fs.String("db-port", "", "external database port")
+	dbName := fs.String("db-name", "", "external database name")
+	dbUser := fs.String("db-user", "", "external database user")
+	dbPassword := fs.String("db-password", os.Getenv("STALWART_MIGRATE_DB_PASSWORD"),
+		"external database password (or set STALWART_MIGRATE_DB_PASSWORD)")
+	sqlDump := fs.String("sql-dump", "", "override the dump file to replay (default: the one this run recorded)")
+
+	yes := fs.Bool("yes", false, "actually perform the rollback; without it, the plan is printed and nothing is touched")
+
+	runID, rest := splitRunID(fs, args)
+	if err := fs.Parse(rest); err != nil {
+		return err
+	}
+	if runID == "" || fs.NArg() != 0 {
+		return fmt.Errorf("usage: stalwart-migrate rollback <run-id> [flags]")
+	}
+
+	store := checkpoint.NewStore(*stateDir)
+	rs, err := store.Load(runID)
+	if err != nil {
+		return fmt.Errorf("load run %s: %w", runID, err)
+	}
+
+	opts := rollback.Options{
+		Deployment: service.Options{
+			Kind: service.Kind(*deployment), UnitName: *unitName, ContainerName: *containerName,
+		},
+		StopTimeout: *stopTimeout, StartTimeout: *startTimeout,
+		DataDir:         *dataDir,
+		BinaryPath:      *binaryPath,
+		ServiceUnitPath: *serviceUnitPath,
+		SQL: backup.SQLOptions{
+			Host: *dbHost, Port: *dbPort, Database: *dbName, User: *dbUser, Password: *dbPassword, OutPath: *sqlDump,
+		},
+		AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword,
+		HTTPClient: &http.Client{}, VerifyTimeout: *verifyTimeout,
+	}
+
+	plan, err := rollback.BuildPlan(rs, opts)
+	if err != nil {
+		return err
+	}
+	fmt.Print(plan.String())
+
+	if !*yes {
+		fmt.Println("\nnothing has been touched. Re-run with --yes to perform this rollback.")
+		return nil
+	}
+	if *adminURL == "" {
+		fmt.Println("\nnote: without --admin-url, the reachability and directory-count checks after the restart are skipped - " +
+			"the rollback will report success on the restore mechanics alone.")
+	}
+
+	fmt.Println("\n--- rollback ---")
+	report, err := rollback.Run(context.Background(), store, rs, opts)
+	fmt.Print(report.String())
+	if err != nil {
+		return fmt.Errorf("rollback did not complete: %w", err)
+	}
+
+	fmt.Printf("\nROLLBACK COMPLETE for run %s: the instance is back on %s. "+
+		"The failed attempt's data and binary are preserved under .failed-%s names, and this run's backups are untouched, "+
+		"so a retry after the underlying issue is fixed doesn't have to re-capture anything.\n",
+		rs.RunID, rs.SourceVersion, rs.RunID)
+	return nil
+}

cmd/stalwart-migrate/run.go

@@ -17,12 +17,10 @@ import (
 )
 
 // runRun implements `stalwart-migrate run`. Only --dry-run is available
-// today: a real cutover needs internal/rollback (so a failed migration can
-// actually be undone) and real systemd/Docker service control, neither of
-// which exist yet - see ARCHITECTURE.md §8. Committing to a real migration
-// without a working rollback would violate the one thing this whole tool
-// exists to guarantee, so `run` without --dry-run refuses rather than doing
-// it partway.
+// today: the cutover phase itself (§4.5) isn't built. internal/rollback and
+// internal/service now exist, so the missing piece is no longer "this can't
+// be undone" but "there's nothing here to undo" - see ARCHITECTURE.md §8.
+// `run` without --dry-run refuses rather than doing a migration partway.
 //
 // --dry-run runs preflight and a real backup (see the caveat printed below
 // about why backup still touches the live data directory), then - if the
@@ -70,8 +68,9 @@ func runRun(args []string) (err error) {
 
 	if !*dryRun {
 		return fmt.Errorf(
-			"real (non-dry-run) migrations aren't available yet: internal/rollback and real systemd/Docker service control " +
-				"aren't implemented, so this tool can't yet guarantee it can undo a failed cutover. Run with --dry-run to " +
+			"real (non-dry-run) migrations aren't available yet: the cutover phase - installing the new binary, rewriting the " +
+				"service definition, and starting the migrated instance for real (ARCHITECTURE.md §4.5) - isn't implemented. " +
+				"Rollback is, so a future cutover will be undoable; there just isn't one to undo yet. Run with --dry-run to " +
 				"validate the migration mechanics against a disposable sandbox copy of your data - nothing in production is touched",
 		)
 	}
@@ -136,9 +135,9 @@ func runRun(args []string) (err error) {
 	fmt.Printf("\nplan: %s\n", p.Reason)
 
 	fmt.Println("\n--- backup ---")
-	fmt.Println("(dry-run does not stop the live Stalwart service itself - no service control is implemented yet. " +
-		"For a guaranteed-consistent snapshot, stop stalwart before running this; otherwise the filesystem copy may " +
-		"reflect a live, in-use store. This is unrelated to whether production gets touched - it never does.)")
+	fmt.Println("(dry-run does not stop the live Stalwart service itself - internal/service can now do that, but dry-run " +
+		"isn't wired to offer it. For a guaranteed-consistent snapshot, stop stalwart before running this; otherwise the " +
+		"filesystem copy may reflect a live, in-use store. This is unrelated to whether production gets touched - it never does.)")
 
 	backupDir := filepath.Join(runWorkDir, "backup")
 	scriptDest := filepath.Join(runWorkDir, "migrate_v016.py")

cmd/stalwart-migrate/status.go

@@ -10,13 +10,21 @@ import (
 func runStatus(args []string) error {
 	fs := flag.NewFlagSet("status", flag.ExitOnError)
 	stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory runs are checkpointed in")
-	if err := fs.Parse(args); err != nil {
+
+	// Same treatment as `rollback`: without this, `status <run-id>
+	// --state-dir X` would look up the run in the default directory and
+	// report it missing, since flag parsing stops at the run-id.
+	runID, rest := splitRunID(fs, args)
+	if err := fs.Parse(rest); err != nil {
 		return err
 	}
+	if fs.NArg() != 0 {
+		return fmt.Errorf("usage: stalwart-migrate status [run-id] [flags]")
+	}
 
 	store := checkpoint.NewStore(*stateDir)
 
-	if fs.NArg() == 0 {
+	if runID == "" {
 		ids, err := store.List()
 		if err != nil {
 			return fmt.Errorf("list runs: %w", err)
@@ -32,7 +40,6 @@ func runStatus(args []string) error {
 		return nil
 	}
 
-	runID := fs.Arg(0)
 	rs, err := store.Load(runID)
 	if err != nil {
 		return fmt.Errorf("load run %s: %w", runID, err)