Rollback was the one phase gating everything else: `run` without --dry-run refused because this tool could not undo a cutover it had committed to. That reason is now gone, and the refusal has narrowed to the fact that there is no real cutover to undo yet. internal/rollback implements ARCHITECTURE.md 4.8 as eight checkpointed steps under PhaseRollback: verify-backup, stop-service, preserve-failed-state, restore-data, restore-binary, restore-service-config, start-service, verify-rollback. Three things depart from what 4.8 specified, each for a reason: - The backup is re-verified against its manifest *before* the service is stopped, which the design didn't call out. Finding a corrupt backup is survivable while the failed instance is still up, and unsurvivable once its data directory has been moved aside. - BuildPlan is separate from Run, so every reason to refuse (closed rollback window, FoundationDB, no recorded backup, unknown deployment kind, missing database credentials) is found before anything is touched. The CLI prints that resolved plan and acts only with --yes. - The restore is re-verified against the same manifest after writing. A restore that put back truncated bytes and reported success would be worse than one that failed outright. Nothing from the failed attempt is deleted: the half-migrated data directory and the displaced binary are moved to .failed-<run-id> names, so a retry after the underlying issue is fixed still has both the evidence and the artifacts. Afterwards a reduced validation suite runs against the *restored* instance (version, reachability, directory counts) rather than assuming the restore worked. internal/service is a new package holding the systemd/Docker control this needs. It's separate rather than living inside internal/rollback because cutover will need the identical operations, and because the commands that can take mail delivery down belong in one auditable place - the same reasoning that makes stalwartapi the only thing speaking JMAP. preflight.DeploymentKind is now a type alias for service.Kind so detection and control can't drift apart. Its Active() reads `systemctl is-active`'s output rather than its exit status: systemctl exits non-zero for every non-active state, so exit-status logic would make "inactive" - the answer a rollback most needs - look like a failure to read the state at all. Also fixes a pre-existing bug in `status`: Go's flag package stops parsing at the first positional argument, so `status <run-id> --state-dir X` looked the run up in the default directory and reported it missing. `rollback` would have inherited the same footgun on a command whose flags decide what gets overwritten. Still open: `confirm` cannot set RollbackWindowClosed. Rollback honours the flag and refuses when it's set, but closing the window is the point of no return for the backups this restores from, so it should land with the retention policy 6 describes rather than before it. Verified end to end against a fake systemd deployment: half-migrated data restored to its original contents, failed state preserved, old binary reinstalled and reporting 0.15.5, unit restarted, and a re-run of the completed rollback inert.
228 lines
6.7 KiB
Go
228 lines
6.7 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestNewRefusesUnknownKind(t *testing.T) {
|
|
for _, kind := range []Kind{Unknown, "", "kubernetes"} {
|
|
if _, err := New(Options{Kind: kind}); err == nil {
|
|
t.Errorf("New(%q): want error, got nil - guessing how to stop Stalwart is exactly what this must not do", kind)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNewDefaultsTargetNames(t *testing.T) {
|
|
systemd, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := systemd.Target(), "systemd unit stalwart"; got != want {
|
|
t.Errorf("Target() = %q, want %q", got, want)
|
|
}
|
|
docker, err := New(Options{Kind: Docker})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := docker.Target(), "docker container stalwart"; got != want {
|
|
t.Errorf("Target() = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSystemdStopStartReloadInvocations(t *testing.T) {
|
|
dir := t.TempDir()
|
|
log := argsFile(t, dir)
|
|
withFakeExecutable(t, "systemctl", fakeScriptLoggingArgs(log, "exit 0"))
|
|
|
|
c, err := New(Options{Kind: Systemd, UnitName: "stalwart-mail"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
if err := c.Stop(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.ReloadConfig(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
want := "stop stalwart-mail\ndaemon-reload\nstart stalwart-mail\n"
|
|
if got := readArgsFile(t, log); got != want {
|
|
t.Errorf("systemctl invocations:\ngot: %q\nwant: %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSystemdStopReportsCommandFailure(t *testing.T) {
|
|
withFakeExecutable(t, "systemctl", "#!/bin/sh\necho 'Failed to stop stalwart.service: Access denied' >&2\nexit 1\n")
|
|
c, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = c.Stop(context.Background())
|
|
if err == nil {
|
|
t.Fatal("Stop: want error when systemctl fails, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "Access denied") {
|
|
t.Errorf("Stop error %q does not carry systemctl's own output, which is the only clue an operator gets", err)
|
|
}
|
|
}
|
|
|
|
// systemctl exits non-zero for every non-active state, so Active has to
|
|
// read its output rather than its exit status - otherwise "inactive", the
|
|
// answer a rollback most needs, would look like a failure to read state.
|
|
func TestSystemdActiveReadsOutputNotExitStatus(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
state string
|
|
exitCode int
|
|
want bool
|
|
}{
|
|
{"active", 0, true},
|
|
{"activating", 3, true},
|
|
{"reloading", 3, true},
|
|
{"deactivating", 3, true}, // still holding the data directory open
|
|
{"inactive", 3, false},
|
|
{"failed", 3, false},
|
|
} {
|
|
t.Run(tc.state, func(t *testing.T) {
|
|
withFakeExecutable(t, "systemctl", fmt.Sprintf("#!/bin/sh\necho %s\nexit %d\n", tc.state, tc.exitCode))
|
|
c, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
active, err := c.Active(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Active() for state %q: unexpected error %v", tc.state, err)
|
|
}
|
|
if active != tc.want {
|
|
t.Errorf("Active() for state %q = %v, want %v", tc.state, active, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSystemdActiveErrorsOnUnrecognizedState(t *testing.T) {
|
|
withFakeExecutable(t, "systemctl", "#!/bin/sh\necho 'command not found'\nexit 127\n")
|
|
c, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := c.Active(context.Background()); err == nil {
|
|
t.Error("Active: want error for unreadable state, got nil - an unreadable state must never collapse into 'not running'")
|
|
}
|
|
}
|
|
|
|
func TestDockerStopStartInvocations(t *testing.T) {
|
|
dir := t.TempDir()
|
|
log := argsFile(t, dir)
|
|
withFakeExecutable(t, "docker", fakeScriptLoggingArgs(log, "exit 0"))
|
|
|
|
c, err := New(Options{Kind: Docker, ContainerName: "mail"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
if err := c.Stop(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.ReloadConfig(ctx); err != nil {
|
|
t.Fatalf("ReloadConfig should be a no-op for docker: %v", err)
|
|
}
|
|
|
|
want := "stop mail\nstart mail\n"
|
|
if got := readArgsFile(t, log); got != want {
|
|
t.Errorf("docker invocations:\ngot: %q\nwant: %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestDockerActive(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
out string
|
|
want bool
|
|
}{{"true", true}, {"false", false}} {
|
|
t.Run(tc.out, func(t *testing.T) {
|
|
withFakeExecutable(t, "docker", fmt.Sprintf("#!/bin/sh\necho %s\n", tc.out))
|
|
c, err := New(Options{Kind: Docker})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
active, err := c.Active(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if active != tc.want {
|
|
t.Errorf("Active() = %v, want %v", active, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDockerActiveErrorsWhenContainerMissing(t *testing.T) {
|
|
withFakeExecutable(t, "docker", "#!/bin/sh\necho 'Error: No such object: stalwart' >&2\nexit 1\n")
|
|
c, err := New(Options{Kind: Docker})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := c.Active(context.Background()); err == nil {
|
|
t.Error("Active: want error when the container doesn't exist, got nil")
|
|
}
|
|
}
|
|
|
|
// WaitFor exists because systemctl and docker both return as soon as the
|
|
// *request* succeeded: this proves it keeps polling past a still-running
|
|
// state rather than accepting the first answer.
|
|
func TestWaitForPollsUntilStateChanges(t *testing.T) {
|
|
dir := t.TempDir()
|
|
counter := dir + "/calls"
|
|
withFakeExecutable(t, "systemctl", fmt.Sprintf(
|
|
"#!/bin/sh\nn=$(cat %[1]q 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > %[1]q\n"+
|
|
"if [ $n -lt 3 ]; then echo deactivating; exit 3; fi\necho inactive; exit 3\n", counter))
|
|
|
|
c, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := WaitFor(context.Background(), c, false, 5*time.Second); err != nil {
|
|
t.Fatalf("WaitFor: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestWaitForTimesOutWhileStillActive(t *testing.T) {
|
|
withFakeExecutable(t, "systemctl", "#!/bin/sh\necho active\n")
|
|
c, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = WaitFor(context.Background(), c, false, 300*time.Millisecond)
|
|
if err == nil {
|
|
t.Fatal("WaitFor: want timeout error while the unit is still active, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "not stopped") {
|
|
t.Errorf("WaitFor error %q should say what it was waiting for", err)
|
|
}
|
|
}
|
|
|
|
func TestWaitForSurfacesLastStateReadError(t *testing.T) {
|
|
withFakeExecutable(t, "systemctl", "#!/bin/sh\necho 'no such unit' >&2\nexit 4\n")
|
|
c, err := New(Options{Kind: Systemd})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = WaitFor(context.Background(), c, false, 300*time.Millisecond)
|
|
if err == nil {
|
|
t.Fatal("WaitFor: want error when the state can't be read at all, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "state couldn't be read") {
|
|
t.Errorf("WaitFor error %q should distinguish 'never reached the state' from 'never could tell'", err)
|
|
}
|
|
}
|