Cut over a container, or refuse to for a reason
A container cannot be edited in place the way a unit file can, so cutting one over means rebuilding it. That makes silent loss the default failure: a container recreated without its capabilities, its custom network or its device mappings starts cleanly and is quietly not the server it was. Section 4.5 already answers this for a unit file -- it rewrites in place rather than regenerating, because a generated unit would drop hardening options this tool has no business having an opinion about, and it refuses to edit a line it only partly understands. The same rule applies here, where the whole definition has to be rebuilt: the parts this understands are carried across, and a container using anything else is refused by name rather than rebuilt without it. The list of what it looks for is conservative and not exhaustive, which is the safe direction: docker's HostConfig has far more fields, and one this does not know about is a reason not to be recreating that container at all. The old container is renamed, not removed, and nothing here prunes the old image. Together they are the container's manual restore path -- one command starts the previous container again -- which is as close to section 4.2's preserved binary as a container gets. The inspect output is preserved as an artifact before anything is replaced, for the reason the unit file is: an operator putting a machine back by hand should not also be reconstructing the definition from memory. Recovery-mode variables are stripped from the recreated container's environment. Leaving STALWART_RECOVERY_MODE set would recovery-boot on every restart, which is the same footgun the unit rewrite exists to prevent. Run now branches, with the health check and quota recalculation shared: those ask the same question whatever started the server. The binary path moved inside an else and is otherwise untouched -- no existing test needed editing, which is the evidence for that. The container path is opt-in through Options.Container, so a Docker deployment without it is still refused exactly as before. Nothing calls it yet; wiring `run` up and lifting preflight's refusal is what remains of #3, and ARCHITECTURE.md says so in both places it previously said Docker was refused outright.
This commit is contained in:
+128
-88
@@ -43,6 +43,12 @@ type Options struct {
|
||||
// moved the old binary aside, so this path is normally empty by now.
|
||||
BinaryPath string
|
||||
|
||||
// Container, when set, cuts over a container deployment instead of a
|
||||
// binary and a unit file. Opt-in: a Docker deployment without it is
|
||||
// still refused, so a caller that has not been taught to stage an image
|
||||
// cannot reach a half-built container path by accident.
|
||||
Container *ContainerOptions
|
||||
|
||||
// ServiceUnitPath is the systemd unit to rewrite. ConfigPath, if set,
|
||||
// becomes the unit's --config argument.
|
||||
ServiceUnitPath string
|
||||
@@ -147,9 +153,19 @@ func BuildPlan(rs *checkpoint.RunState, opts Options) (Plan, error) {
|
||||
kind = service.Kind(rs.Topology.DeploymentKind)
|
||||
}
|
||||
if kind == service.Docker {
|
||||
return p, fmt.Errorf(
|
||||
"cutover: this run's deployment is a Docker container, where cutting over means pulling a new image and recreating the " +
|
||||
"container rather than swapping a binary and rewriting a unit. This tool doesn't automate that - do it by hand")
|
||||
if opts.Container == nil {
|
||||
return p, fmt.Errorf(
|
||||
"cutover: this run's deployment is a Docker container, where cutting over means pulling a new image and recreating the " +
|
||||
"container rather than swapping a binary and rewriting a unit. No container options were given, so there is nothing " +
|
||||
"to recreate it from - stage the target image and pass them, or do it by hand")
|
||||
}
|
||||
if opts.Container.StagedImage == "" {
|
||||
return p, fmt.Errorf("cutover: no staged image to cut over to - stage the target image first")
|
||||
}
|
||||
// The binary checks below are about a file and a unit, neither of
|
||||
// which a container has.
|
||||
p.Target = "docker container " + opts.Container.ContainerName
|
||||
return p, nil
|
||||
}
|
||||
deployment := opts.Deployment
|
||||
deployment.Kind = kind
|
||||
@@ -229,103 +245,118 @@ func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState,
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := step("verify-staged-binary", func() (checkpoint.StepOutcome, error) {
|
||||
got, err := preflight.DetectVersion(ctx, plan.StagedBinaryPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("couldn't read the staged binary's version: %w", err)
|
||||
// A container is replaced rather than edited, so none of the binary
|
||||
// path applies: there is no file to install and no unit to rewrite.
|
||||
// What follows either branch - confirming it answers, and the quota
|
||||
// recalculation - is the same question whatever started it.
|
||||
if opts.Container != nil {
|
||||
if err := runContainerCutover(ctx, rs, step, *opts.Container); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if rs.TargetVersion != "" && got != rs.TargetVersion {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf(
|
||||
"staged binary %s reports version %s, but this run targets %s - installing it would migrate to a version nobody planned for",
|
||||
plan.StagedBinaryPath, got, rs.TargetVersion)
|
||||
if err := step("wait-running", func() (checkpoint.StepOutcome, error) {
|
||||
if err := service.WaitFor(ctx, controller, true, startTimeoutOr(opts)); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
} else {
|
||||
if err := step("verify-staged-binary", func() (checkpoint.StepOutcome, error) {
|
||||
got, err := preflight.DetectVersion(ctx, plan.StagedBinaryPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("couldn't read the staged binary's version: %w", err)
|
||||
}
|
||||
if rs.TargetVersion != "" && got != rs.TargetVersion {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf(
|
||||
"staged binary %s reports version %s, but this run targets %s - installing it would migrate to a version nobody planned for",
|
||||
plan.StagedBinaryPath, got, rs.TargetVersion)
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("staged binary %s reports %s, matching this run's target", plan.StagedBinaryPath, got), Extra: got}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("staged binary %s reports %s, matching this run's target", plan.StagedBinaryPath, got), Extra: got}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := step("install-binary", func() (checkpoint.StepOutcome, error) {
|
||||
sum, size, err := installBinary(plan.StagedBinaryPath, plan.BinaryPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
if err := step("install-binary", func() (checkpoint.StepOutcome, error) {
|
||||
sum, size, err := installBinary(plan.StagedBinaryPath, plan.BinaryPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
rs.RecordArtifact(ArtifactNewBinary, checkpoint.Artifact{Path: plan.BinaryPath, SHA256: sum, SizeBytes: size})
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%d bytes)", plan.StagedBinaryPath, plan.BinaryPath, size)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
rs.RecordArtifact(ArtifactNewBinary, checkpoint.Artifact{Path: plan.BinaryPath, SHA256: sum, SizeBytes: size})
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%d bytes)", plan.StagedBinaryPath, plan.BinaryPath, size)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := step("install-config", func() (checkpoint.StepOutcome, error) {
|
||||
if plan.ConfigSource == "" {
|
||||
if err := step("install-config", func() (checkpoint.StepOutcome, error) {
|
||||
if plan.ConfigSource == "" {
|
||||
return checkpoint.StepOutcome{
|
||||
Verdict: string(StatusSkipped),
|
||||
Detail: "no converted config to install - the unit is repointed at whatever is already at ConfigPath",
|
||||
}, nil
|
||||
}
|
||||
owner, err := installConfig(plan.ConfigSource, plan.ConfigPath, opts.ConfigOwnerReference)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%s)", plan.ConfigSource, plan.ConfigPath, owner)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := step("update-service-definition", func() (checkpoint.StepOutcome, error) {
|
||||
preserved, err := preserveUnit(plan.ServiceUnitPath, rs.RunID)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
sum, size, err := hashFile(preserved)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
// Recorded before the rewrite, so a crash between preserving and
|
||||
// rewriting still leaves the original findable.
|
||||
rs.RecordArtifact(ArtifactServiceUnit, checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size})
|
||||
|
||||
original, err := os.ReadFile(preserved)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("cutover: read preserved unit %s: %w", preserved, err)
|
||||
}
|
||||
rewritten, err := RewriteUnit(string(original), plan.BinaryPath, plan.ConfigPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
if err := writeFileAtomic(plan.ServiceUnitPath, []byte(rewritten), 0o644); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{
|
||||
Verdict: string(StatusSkipped),
|
||||
Detail: "no converted config to install - the unit is repointed at whatever is already at ConfigPath",
|
||||
Detail: fmt.Sprintf("pointed %s at %s; the original is preserved at %s", plan.ServiceUnitPath, plan.BinaryPath, preserved),
|
||||
Extra: preserved,
|
||||
}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
owner, err := installConfig(plan.ConfigSource, plan.ConfigPath, opts.ConfigOwnerReference)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%s)", plan.ConfigSource, plan.ConfigPath, owner)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := step("update-service-definition", func() (checkpoint.StepOutcome, error) {
|
||||
preserved, err := preserveUnit(plan.ServiceUnitPath, rs.RunID)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
if err := step("reload-service-definition", func() (checkpoint.StepOutcome, error) {
|
||||
if err := controller.ReloadConfig(ctx); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: "service manager re-read the updated definition"}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
sum, size, err := hashFile(preserved)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
// Recorded before the rewrite, so a crash between preserving and
|
||||
// rewriting still leaves the original findable.
|
||||
rs.RecordArtifact(ArtifactServiceUnit, checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size})
|
||||
|
||||
original, err := os.ReadFile(preserved)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("cutover: read preserved unit %s: %w", preserved, err)
|
||||
startTimeout := startTimeoutOr(opts)
|
||||
if err := step("start-service", func() (checkpoint.StepOutcome, error) {
|
||||
if err := controller.Start(ctx); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
if err := service.WaitFor(ctx, controller, true, startTimeout); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
rewritten, err := RewriteUnit(string(original), plan.BinaryPath, plan.ConfigPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
if err := writeFileAtomic(plan.ServiceUnitPath, []byte(rewritten), 0o644); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{
|
||||
Detail: fmt.Sprintf("pointed %s at %s; the original is preserved at %s", plan.ServiceUnitPath, plan.BinaryPath, preserved),
|
||||
Extra: preserved,
|
||||
}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := step("reload-service-definition", func() (checkpoint.StepOutcome, error) {
|
||||
if err := controller.ReloadConfig(ctx); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: "service manager re-read the updated definition"}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
startTimeout := opts.StartTimeout
|
||||
if startTimeout <= 0 {
|
||||
startTimeout = 60 * time.Second
|
||||
}
|
||||
if err := step("start-service", func() (checkpoint.StepOutcome, error) {
|
||||
if err := controller.Start(ctx); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
if err := service.WaitFor(ctx, controller, true, startTimeout); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
healthTimeout := opts.HealthTimeout
|
||||
@@ -587,3 +618,12 @@ func installConfig(src, dst, reference string) (ownership string, err error) {
|
||||
}
|
||||
return fmt.Sprintf("mode %v, ownership unchanged (no reference file to copy it from)", perm), nil
|
||||
}
|
||||
|
||||
// startTimeoutOr is the wait for the service to come up, defaulted. Shared
|
||||
// because both branches wait for the same thing.
|
||||
func startTimeoutOr(opts Options) time.Duration {
|
||||
if opts.StartTimeout > 0 {
|
||||
return opts.StartTimeout
|
||||
}
|
||||
return 60 * time.Second
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user