Initial commit: stalwart-migrator design and scaffolding
In-place upgrade tool for Stalwart Mail Server (0.15.5 -> latest) with checkpointed rollback and post-migration validation. Design stage; see ARCHITECTURE.md.
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// ApplyOptions configures stalwart-cli apply invocations against a
|
||||
// recovery-mode instance - see UPGRADING/v0_16.md's documented
|
||||
// STALWART_URL/STALWART_USER/STALWART_PASSWORD + `stalwart-cli apply --file`
|
||||
// sequence.
|
||||
type ApplyOptions struct {
|
||||
CLIBinaryPath string // defaults to "stalwart-cli"
|
||||
URL string
|
||||
User string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Apply runs `stalwart-cli apply --file <file>` once, with credentials
|
||||
// passed as environment variables exactly as the upgrade guide's own
|
||||
// example does, rather than on the command line where they'd be visible to
|
||||
// anything reading this process's argv.
|
||||
func Apply(ctx context.Context, o ApplyOptions, file string) error {
|
||||
binary := o.CLIBinaryPath
|
||||
if binary == "" {
|
||||
binary = "stalwart-cli"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, binary, "apply", "--file", file)
|
||||
cmd.Env = append(os.Environ(),
|
||||
"STALWART_URL="+o.URL,
|
||||
"STALWART_USER="+o.User,
|
||||
"STALWART_PASSWORD="+o.Password,
|
||||
)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("recovery: stalwart-cli apply --file %s failed: %w (output: %s)", file, err, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ApplyAll runs Apply for each file in order - export.json first, then any
|
||||
// additional test-deployment snapshots (ARCHITECTURE.md §4.3/§4.4) -
|
||||
// stopping at the first failure so a partial, silently-incomplete settings
|
||||
// replay never gets reported as success.
|
||||
func ApplyAll(ctx context.Context, o ApplyOptions, files []string) error {
|
||||
for i, f := range files {
|
||||
if err := Apply(ctx, o, f); err != nil {
|
||||
return fmt.Errorf("applied %d/%d file(s) before failing: %w", i, len(files), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestApplyInvokesCLIWithEnvCredentials(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
withFakeExecutable(t, "stalwart-cli", fmt.Sprintf("#!/bin/sh\necho \"$@ URL=$STALWART_URL USER=$STALWART_USER\" >> %q\nexit 0\n", log))
|
||||
|
||||
err := Apply(context.Background(), ApplyOptions{URL: "http://127.0.0.1:8080", User: "admin", Password: "secret"}, "/tmp/export.json")
|
||||
if err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
for _, want := range []string{"apply", "--file /tmp/export.json", "URL=http://127.0.0.1:8080", "USER=admin"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("stalwart-cli invoked with %q, missing %q", got, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "secret") {
|
||||
t.Error("password should not appear in argv")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPropagatesFailure(t *testing.T) {
|
||||
withFakeExecutable(t, "stalwart-cli", "#!/bin/sh\necho 'invalid object' >&2\nexit 1\n")
|
||||
err := Apply(context.Background(), ApplyOptions{URL: "http://127.0.0.1:8080", User: "admin", Password: "x"}, "/tmp/export.json")
|
||||
if err == nil {
|
||||
t.Fatal("Apply should error when stalwart-cli exits non-zero")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "invalid object") {
|
||||
t.Errorf("error = %v, want it to include stderr", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAllStopsAtFirstFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
script := fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\ncase \"$*\" in\n *bad.json*) exit 1 ;;\n *) exit 0 ;;\nesac\n", log)
|
||||
withFakeExecutable(t, "stalwart-cli", script)
|
||||
|
||||
err := ApplyAll(context.Background(), ApplyOptions{URL: "http://127.0.0.1:8080", User: "admin", Password: "x"},
|
||||
[]string{filepath.Join(dir, "good1.json"), filepath.Join(dir, "bad.json"), filepath.Join(dir, "good2.json")})
|
||||
if err == nil {
|
||||
t.Fatal("ApplyAll should fail on bad.json")
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
if strings.Contains(got, "good2.json") {
|
||||
t.Error("ApplyAll should stop before applying good2.json after bad.json failed")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "1/3") {
|
||||
t.Errorf("error = %v, want it to report 1/3 files applied before failing", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
// Package recovery implements supervision of the Stalwart recovery-mode process and the settings apply step.
|
||||
// See ARCHITECTURE.md §4.4 for the design.
|
||||
package recovery
|
||||
@@ -0,0 +1,39 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// withFakeExecutable puts a fake executable named `name` at the front of
|
||||
// PATH for the duration of the test, so code that shells out to a
|
||||
// real-world tool (stalwart-cli) can be exercised without that tool
|
||||
// actually being installed. t.Setenv restores PATH automatically.
|
||||
func withFakeExecutable(t *testing.T, name, script string) (dir string) {
|
||||
t.Helper()
|
||||
dir = t.TempDir()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
return dir
|
||||
}
|
||||
|
||||
func argsFile(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
return filepath.Join(dir, "invoked-args.log")
|
||||
}
|
||||
|
||||
func readArgsFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WaitForHealthy polls url until it responds (any status - even 401 proves
|
||||
// the HTTP server itself is up and routing requests, which is what this
|
||||
// check exists to confirm) or timeout elapses, returning a descriptive
|
||||
// error on timeout rather than hanging indefinitely. This is what makes
|
||||
// recovery mode's startup supervised rather than fire-and-forget - see
|
||||
// ARCHITECTURE.md §4.4.
|
||||
func WaitForHealthy(ctx context.Context, httpClient *http.Client, url string, timeout time.Duration) error {
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 5 * time.Second}
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastErr error
|
||||
for time.Now().Before(deadline) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("recovery: %s did not become reachable within %s: %w", url, timeout, lastErr)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMain lets this test binary also act as a fake Stalwart binary for
|
||||
// subprocess tests - the standard os/exec "helper process" technique (see
|
||||
// Go's own os/exec_test.go). When STALWART_MIGRATOR_TEST_HELPER=1 is set,
|
||||
// the binary runs a minimal HTTP server on STALWART_MIGRATOR_TEST_PORT
|
||||
// until it receives SIGTERM (or, if STALWART_MIGRATOR_TEST_IGNORE_SIGTERM=1,
|
||||
// ignores SIGTERM to exercise the SIGKILL escalation path) instead of
|
||||
// running the actual test suite. This avoids needing a real Stalwart binary
|
||||
// - or a fabricated stand-in for its HTTP behavior - anywhere in these
|
||||
// tests: it's real net/http and real process signaling, just running under
|
||||
// this package's own compiled binary.
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("STALWART_MIGRATOR_TEST_HELPER") == "1" {
|
||||
runFakeStalwartServer()
|
||||
return
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func runFakeStalwartServer() {
|
||||
port := os.Getenv("STALWART_MIGRATOR_TEST_PORT")
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:"+port)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "fake stalwart: listen:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})}
|
||||
go srv.Serve(ln)
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
if os.Getenv("STALWART_MIGRATOR_TEST_IGNORE_SIGTERM") == "1" {
|
||||
signal.Ignore(syscall.SIGTERM)
|
||||
select {} // block forever; the test must SIGKILL this process itself
|
||||
}
|
||||
signal.Notify(sigCh, syscall.SIGTERM)
|
||||
<-sigCh
|
||||
os.Exit(0)
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProcessOptions configures how the target binary is launched.
|
||||
type ProcessOptions struct {
|
||||
BinaryPath string
|
||||
ConfigPath string
|
||||
|
||||
// RecoveryMode, when true, sets STALWART_RECOVERY_MODE=1 and
|
||||
// STALWART_RECOVERY_ADMIN=<AdminUser>:<AdminPassword> - exactly the
|
||||
// environment variables Stalwart's own upgrade guide uses to bring the
|
||||
// new binary up in recovery mode against a not-yet-migrated store (see
|
||||
// UPGRADING/v0_16.md). When false, the binary is started as a normal
|
||||
// boot against ConfigPath - used after recovery mode to confirm the
|
||||
// migrated store comes up cleanly under an ordinary start, not just a
|
||||
// recovery one (see ARCHITECTURE.md's dry-run design).
|
||||
RecoveryMode bool
|
||||
AdminUser string
|
||||
AdminPassword string
|
||||
|
||||
// ExtraEnv is appended after any recovery-mode env vars. This is what
|
||||
// lets a dry-run point the process at a sandbox without RecoveryMode's
|
||||
// two env vars becoming the only way to parameterize the child process.
|
||||
ExtraEnv []string
|
||||
}
|
||||
|
||||
// Process supervises one run of the target Stalwart binary as a background
|
||||
// child process, so a caller can start it, wait for it to become healthy,
|
||||
// interact with it, and stop it again - without ever touching a real
|
||||
// systemd unit or Docker container. See ARCHITECTURE.md §4.4.
|
||||
type Process struct {
|
||||
cmd *exec.Cmd
|
||||
}
|
||||
|
||||
// Start launches the binary. It returns as soon as the OS has started the
|
||||
// process - it does not wait for Stalwart itself to become ready; use
|
||||
// WaitForHealthy for that.
|
||||
func (p *Process) Start(ctx context.Context, o ProcessOptions) error {
|
||||
var env []string
|
||||
if o.RecoveryMode {
|
||||
env = append(env,
|
||||
"STALWART_RECOVERY_MODE=1",
|
||||
fmt.Sprintf("STALWART_RECOVERY_ADMIN=%s:%s", o.AdminUser, o.AdminPassword),
|
||||
)
|
||||
}
|
||||
env = append(env, o.ExtraEnv...)
|
||||
|
||||
cmd := exec.CommandContext(ctx, o.BinaryPath, "--config", o.ConfigPath)
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("recovery: start %s: %w", o.BinaryPath, err)
|
||||
}
|
||||
p.cmd = cmd
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop sends SIGTERM and waits up to gracePeriod for the process to exit -
|
||||
// mirroring the upgrade guide's own "Ctrl+C in the first terminal" step,
|
||||
// just automated - escalating to SIGKILL if it doesn't exit in time so a
|
||||
// stuck child process can never hang a migration run indefinitely. Safe to
|
||||
// call on a Process that was never successfully started.
|
||||
func (p *Process) Stop(gracePeriod time.Duration) error {
|
||||
if p.cmd == nil || p.cmd.Process == nil {
|
||||
return nil
|
||||
}
|
||||
if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
||||
return fmt.Errorf("recovery: signal process (pid %d): %w", p.cmd.Process.Pid, err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- p.cmd.Wait() }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
// A non-zero exit from a SIGTERM-based shutdown is expected and not
|
||||
// itself a failure worth reporting - only an unexpected Wait error is.
|
||||
if err != nil {
|
||||
if _, ok := err.(*exec.ExitError); !ok {
|
||||
return fmt.Errorf("recovery: wait for process (pid %d): %w", p.cmd.Process.Pid, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
case <-time.After(gracePeriod):
|
||||
_ = p.cmd.Process.Kill()
|
||||
<-done
|
||||
return fmt.Errorf("recovery: process (pid %d) did not exit within %s of SIGTERM - sent SIGKILL", p.cmd.Process.Pid, gracePeriod)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// freePort asks the OS for an unused TCP port by binding to :0 and
|
||||
// immediately releasing it. There's a small window where something else
|
||||
// could grab it before the fake server binds, but that's an acceptable,
|
||||
// standard tradeoff for tests.
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
return ln.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
func helperProcessEnv(port int, extra ...string) []string {
|
||||
env := []string{
|
||||
"STALWART_MIGRATOR_TEST_HELPER=1",
|
||||
fmt.Sprintf("STALWART_MIGRATOR_TEST_PORT=%d", port),
|
||||
}
|
||||
return append(env, extra...)
|
||||
}
|
||||
|
||||
func testBinaryPath(t *testing.T) string {
|
||||
t.Helper()
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return self
|
||||
}
|
||||
|
||||
func TestProcessStartWaitHealthyStop(t *testing.T) {
|
||||
port := freePort(t)
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
os.WriteFile(configPath, []byte("{}"), 0o644)
|
||||
|
||||
proc := &Process{}
|
||||
err := proc.Start(context.Background(), ProcessOptions{
|
||||
BinaryPath: testBinaryPath(t),
|
||||
ConfigPath: configPath,
|
||||
ExtraEnv: helperProcessEnv(port),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("http://127.0.0.1:%d/", port)
|
||||
if err := WaitForHealthy(context.Background(), nil, url, 5*time.Second); err != nil {
|
||||
t.Fatalf("WaitForHealthy: %v", err)
|
||||
}
|
||||
|
||||
if err := proc.Stop(5 * time.Second); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessStopEscalatesToSIGKILL(t *testing.T) {
|
||||
port := freePort(t)
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
os.WriteFile(configPath, []byte("{}"), 0o644)
|
||||
|
||||
proc := &Process{}
|
||||
err := proc.Start(context.Background(), ProcessOptions{
|
||||
BinaryPath: testBinaryPath(t),
|
||||
ConfigPath: configPath,
|
||||
ExtraEnv: helperProcessEnv(port, "STALWART_MIGRATOR_TEST_IGNORE_SIGTERM=1"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("http://127.0.0.1:%d/", port)
|
||||
if err := WaitForHealthy(context.Background(), nil, url, 5*time.Second); err != nil {
|
||||
t.Fatalf("WaitForHealthy: %v", err)
|
||||
}
|
||||
|
||||
err = proc.Stop(500 * time.Millisecond)
|
||||
if err == nil {
|
||||
t.Fatal("Stop should report an error when it had to escalate to SIGKILL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForHealthyTimesOut(t *testing.T) {
|
||||
// Nothing listens on this port.
|
||||
port := freePort(t)
|
||||
url := fmt.Sprintf("http://127.0.0.1:%d/", port)
|
||||
|
||||
err := WaitForHealthy(context.Background(), nil, url, 500*time.Millisecond)
|
||||
if err == nil {
|
||||
t.Fatal("WaitForHealthy should time out when nothing is listening")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/johnellis/stalwart-migrator/internal/checkpoint"
|
||||
)
|
||||
|
||||
// Options configures one full recovery-mode migration cycle: starting the
|
||||
// target binary in recovery mode, waiting for it to come up, applying the
|
||||
// settings snapshot(s), and stopping it again. See ARCHITECTURE.md §4.4.
|
||||
type Options struct {
|
||||
BinaryPath string
|
||||
ConfigPath string
|
||||
ListenURL string // recovery mode's own HTTP listener, e.g. "http://127.0.0.1:8080"
|
||||
AdminUser string
|
||||
ApplyFiles []string
|
||||
CLIBinaryPath string
|
||||
ExtraEnv []string // lets a dry-run point ports/paths at a sandbox without touching production config
|
||||
StartupTimeout time.Duration
|
||||
StopGrace time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// GenerateRecoveryPassword returns a fresh random one-time password for
|
||||
// STALWART_RECOVERY_ADMIN - never the operator's real admin password, never
|
||||
// logged, never reused across runs or persisted to the checkpoint.
|
||||
func GenerateRecoveryPassword() (string, error) {
|
||||
b := make([]byte, 20)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", fmt.Errorf("recovery: generate password: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// Run executes one recovery-mode cycle as a single checkpointed step. It is
|
||||
// deliberately not decomposed into per-sub-step checkpoints the way
|
||||
// preflight and backup are: if this tool's own process crashes mid-cycle,
|
||||
// the child Stalwart process it started may or may not still be running
|
||||
// independently, and blindly "resuming" by reattaching to a guessed PID or
|
||||
// killing an unrelated process on the recovery port would be more dangerous
|
||||
// than just retrying cleanly. A retry that hits "address already in use"
|
||||
// surfaces the real problem (an orphaned process from the failed attempt)
|
||||
// for a human to clear, rather than this tool guessing at cleanup.
|
||||
//
|
||||
// Whatever happens after Start succeeds, Stop is always attempted on the
|
||||
// way out (via a deferred call), so a failure partway through this cycle
|
||||
// doesn't leak the child process within a single invocation.
|
||||
func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (Report, error) {
|
||||
var report Report
|
||||
|
||||
outcome, err := store.RunStep(rs, checkpoint.PhaseRecovery, "recovery-cycle", func() (out checkpoint.StepOutcome, err error) {
|
||||
password, err := GenerateRecoveryPassword()
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
|
||||
proc := &Process{}
|
||||
if startErr := proc.Start(ctx, ProcessOptions{
|
||||
BinaryPath: opts.BinaryPath, ConfigPath: opts.ConfigPath,
|
||||
RecoveryMode: true, AdminUser: opts.AdminUser, AdminPassword: password,
|
||||
ExtraEnv: opts.ExtraEnv,
|
||||
}); startErr != nil {
|
||||
return checkpoint.StepOutcome{}, startErr
|
||||
}
|
||||
|
||||
stopGrace := opts.StopGrace
|
||||
if stopGrace <= 0 {
|
||||
stopGrace = 10 * time.Second
|
||||
}
|
||||
defer func() {
|
||||
if stopErr := proc.Stop(stopGrace); stopErr != nil && err == nil {
|
||||
err = stopErr
|
||||
}
|
||||
}()
|
||||
|
||||
startupTimeout := opts.StartupTimeout
|
||||
if startupTimeout <= 0 {
|
||||
startupTimeout = 60 * time.Second
|
||||
}
|
||||
if healthErr := WaitForHealthy(ctx, opts.HTTPClient, opts.ListenURL, startupTimeout); healthErr != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("recovery mode did not come up: %w", healthErr)
|
||||
}
|
||||
|
||||
if applyErr := ApplyAll(ctx, ApplyOptions{
|
||||
CLIBinaryPath: opts.CLIBinaryPath, URL: opts.ListenURL, User: opts.AdminUser, Password: password,
|
||||
}, opts.ApplyFiles); applyErr != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("settings apply failed: %w", applyErr)
|
||||
}
|
||||
|
||||
return checkpoint.StepOutcome{
|
||||
Detail: fmt.Sprintf("recovery mode came up at %s, applied %d settings file(s), stopped cleanly", opts.ListenURL, len(opts.ApplyFiles)),
|
||||
}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
report.Results = append(report.Results, CheckResult{Name: "recovery-cycle", Status: StatusFail, Detail: err.Error()})
|
||||
return report, err
|
||||
}
|
||||
report.Results = append(report.Results, CheckResult{Name: "recovery-cycle", Status: StatusOK, Detail: outcome.Detail})
|
||||
return report, nil
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/johnellis/stalwart-migrator/internal/checkpoint"
|
||||
)
|
||||
|
||||
func TestRecoveryRunEndToEndAndResume(t *testing.T) {
|
||||
port := freePort(t)
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
os.WriteFile(configPath, []byte("{}"), 0o644)
|
||||
|
||||
applyDir := t.TempDir()
|
||||
applyLog := argsFile(t, applyDir)
|
||||
withFakeExecutable(t, "stalwart-cli", fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\nexit 0\n", applyLog))
|
||||
|
||||
exportFile := filepath.Join(t.TempDir(), "export.json")
|
||||
os.WriteFile(exportFile, []byte("{}"), 0o644)
|
||||
|
||||
store := checkpoint.NewStore(t.TempDir())
|
||||
rs, err := store.Create("0.15.5", "0.16.14")
|
||||
if err != nil {
|
||||
t.Fatalf("store.Create: %v", err)
|
||||
}
|
||||
|
||||
opts := Options{
|
||||
BinaryPath: testBinaryPath(t),
|
||||
ConfigPath: configPath,
|
||||
ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port),
|
||||
AdminUser: "admin",
|
||||
ApplyFiles: []string{exportFile},
|
||||
ExtraEnv: helperProcessEnv(port),
|
||||
StartupTimeout: 5 * time.Second,
|
||||
StopGrace: 5 * time.Second,
|
||||
}
|
||||
|
||||
report, err := Run(context.Background(), store, rs, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Run #1: %v", err)
|
||||
}
|
||||
if len(report.Results) != 1 || report.Results[0].Status != StatusOK {
|
||||
t.Fatalf("Run #1 report = %+v, want a single OK result", report.Results)
|
||||
}
|
||||
if !rs.Done(checkpoint.PhaseRecovery, "recovery-cycle") {
|
||||
t.Fatal("recovery-cycle should be marked done after a successful run")
|
||||
}
|
||||
if got := readArgsFile(t, applyLog); got == "" {
|
||||
t.Fatal("stalwart-cli apply was never invoked")
|
||||
}
|
||||
|
||||
// -- resume: break the CLI so a re-invocation would fail loudly, and use
|
||||
// -- a port nothing listens on, so re-starting the process would time out.
|
||||
withFakeExecutable(t, "stalwart-cli", "#!/bin/sh\necho should-not-run-again >&2\nexit 1\n")
|
||||
badPort := freePort(t)
|
||||
resumedOpts := opts
|
||||
resumedOpts.ListenURL = fmt.Sprintf("http://127.0.0.1:%d/", badPort)
|
||||
resumedOpts.ExtraEnv = helperProcessEnv(badPort)
|
||||
resumedOpts.StartupTimeout = 300 * time.Millisecond
|
||||
|
||||
resumed, err := store.Load(rs.RunID)
|
||||
if err != nil {
|
||||
t.Fatalf("store.Load (resume): %v", err)
|
||||
}
|
||||
report2, err := Run(context.Background(), store, resumed, resumedOpts)
|
||||
if err != nil {
|
||||
t.Fatalf("Run #2 (resume) should succeed without redoing the cycle: %v", err)
|
||||
}
|
||||
if len(report2.Results) != 1 || report2.Results[0].Detail != report.Results[0].Detail {
|
||||
t.Errorf("resumed report = %+v, want the cached outcome from Run #1", report2.Results)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRunFailsWhenProcessNeverBecomesHealthy(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
os.WriteFile(configPath, []byte("{}"), 0o644)
|
||||
|
||||
// Nothing listens on this port - the binary never actually starts an
|
||||
// HTTP server, since BinaryPath here is a script that just exits.
|
||||
binDir := t.TempDir()
|
||||
binPath := filepath.Join(binDir, "stalwart")
|
||||
os.WriteFile(binPath, []byte("#!/bin/sh\nsleep 5\n"), 0o755)
|
||||
port := freePort(t)
|
||||
|
||||
store := checkpoint.NewStore(t.TempDir())
|
||||
rs, err := store.Create("0.15.5", "0.16.14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
opts := Options{
|
||||
BinaryPath: binPath,
|
||||
ConfigPath: configPath,
|
||||
ListenURL: fmt.Sprintf("http://127.0.0.1:%d/", port),
|
||||
AdminUser: "admin",
|
||||
ApplyFiles: []string{},
|
||||
StartupTimeout: 300 * time.Millisecond,
|
||||
StopGrace: 2 * time.Second,
|
||||
}
|
||||
|
||||
_, err = Run(context.Background(), store, rs, opts)
|
||||
if err == nil {
|
||||
t.Fatal("Run should fail when the process never becomes healthy")
|
||||
}
|
||||
if rs.Status(checkpoint.PhaseRecovery, "recovery-cycle") != checkpoint.StepFailed {
|
||||
t.Errorf("step status = %s, want failed (so a retry is possible)", rs.Status(checkpoint.PhaseRecovery, "recovery-cycle"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusOK Status = "ok"
|
||||
StatusFail Status = "fail"
|
||||
)
|
||||
|
||||
type CheckResult struct {
|
||||
Name string
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Results []CheckResult
|
||||
}
|
||||
|
||||
func (r Report) String() string {
|
||||
var b strings.Builder
|
||||
for _, res := range r.Results {
|
||||
fmt.Fprintf(&b, "[%-4s] %-16s %s\n", strings.ToUpper(string(res.Status)), res.Name, res.Detail)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user