From 15fdb171fe4e8404aaf75b71ec7afec6e337e880 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 11 Nov 2026 23:32:21 -0700 Subject: [PATCH] Add inbuxa cluster join: configure a node into a cluster Topology version 2 adds an optional cluster block to a server machine: where this node reaches the shared SQL store, the coordinator and the blob store, what role it takes, and what it must start after. Every address in it is the operator's; the installer bakes in none of its own, and a file with no cluster block is read exactly as before. inbuxa cluster join -f FILE acts on this machine's block: it installs the secrets every node reads from disk, points the server at the shared store (keeping the old config.json aside for rollback), writes the hostname, role and boot-order drop-ins, and restarts. --dry-run prints every file it would write and stops. It is the node's half only; the shared registry that names the blob store and coordinator is set up once per cluster, not by each node that joins. --- cmd/inbuxa/main.go | 117 ++++++++- internal/cluster/join.go | 392 ++++++++++++++++++++++++++++++ internal/cluster/join_test.go | 228 +++++++++++++++++ internal/state/state.go | 6 + internal/topology/cluster_test.go | 84 +++++++ internal/topology/topology.go | 153 ++++++++++-- 6 files changed, 959 insertions(+), 21 deletions(-) create mode 100644 internal/cluster/join.go create mode 100644 internal/cluster/join_test.go create mode 100644 internal/topology/cluster_test.go diff --git a/cmd/inbuxa/main.go b/cmd/inbuxa/main.go index 5fc3f19..e73fb26 100644 --- a/cmd/inbuxa/main.go +++ b/cmd/inbuxa/main.go @@ -67,6 +67,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine inbuxa cluster doctor [flags] whether this node can reach a cluster's data layer (PostgreSQL, NATS, etcd, Patroni, S3) + inbuxa cluster join -f FILE + point this node's server at the shared store in + the file's cluster block and give it its role inbuxa version this program's version install flags: @@ -144,6 +147,15 @@ checked, so it fits any deployment; omit them all and it does nothing): --s3 HOST:PORT the S3 blob store, checked at /health --timeout DURATION how long each probe waits; default 5s +cluster join flags: + -f FILE the version 2 topology file; this machine's + cluster block says where its store, coordinator + and blob store are and what role it takes + --machine NAME which machine in the file this is; default: the + hostname + --dry-run print what it would write and stop + --yes do not ask for confirmation + deps flags: --server/--console/--webmail SHAPE the shapes to work out the needs for --install do it, rather than only saying what it would do @@ -1060,20 +1072,121 @@ func relayCmd(args []string) int { func clusterCmd(args []string) int { if len(args) == 0 { - fmt.Fprint(os.Stderr, "inbuxa cluster doctor [flags]\n\n"+usage) + fmt.Fprint(os.Stderr, "inbuxa cluster doctor|join [flags]\n\n"+usage) return 2 } switch args[0] { case "doctor": return clusterDoctor(args[1:]) + case "join": + return clusterJoin(args[1:]) case "-h", "--help", "help": fmt.Print(usage) return 0 } - fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor\n", args[0]) + fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor, join\n", args[0]) return 2 } +// clusterJoin points this node's server at the shared store named in the +// topology file's cluster block for this machine, places the secrets every +// node reads from disk, gives the node its role and boot order, and restarts +// it. It never reaches another machine: the file is copied across and run on +// each node, as the rest of the installer works. The registry that names the +// blob store and coordinator is set up once for the cluster, not here. +func clusterJoin(args []string) int { + fs := flag.NewFlagSet("cluster join", flag.ContinueOnError) + fs.Usage = func() { fmt.Print(usage) } + var ( + file = fs.String("f", "", "") + machineName = fs.String("machine", "", "") + dryRun = fs.Bool("dry-run", false, "") + yes = fs.Bool("yes", false, "") + ) + if err := fs.Parse(args); err != nil { + return 2 + } + if *file == "" { + fmt.Fprintln(os.Stderr, "which file? pass -f topology.json") + return 2 + } + t, err := topology.Load(*file) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + return 1 + } + name := *machineName + if name == "" { + name, _ = os.Hostname() + } + m, ok := t.Machine(name) + if !ok { + fmt.Fprintf(os.Stderr, "this machine is %q, which the file does not mention.\n", name) + fmt.Fprintf(os.Stderr, "It describes: %s\n", strings.Join(machineNames(t), ", ")) + fmt.Fprintln(os.Stderr, "Pass --machine to say which one this is.") + return 1 + } + if !m.Clustered() { + fmt.Fprintf(os.Stderr, "machine %q has no cluster block in this file; there is nothing to join.\n", name) + return 1 + } + if m.Shape("server") != "host" { + fmt.Fprintf(os.Stderr, "a cluster node runs the mail server on the host; machine %q runs it as %q.\n", name, m.Shape("server")) + return 1 + } + + etc := filepath.Join(m.Dir, "server") + cfg := cluster.Config{ + Node: *m.Cluster, + Machine: name, + EtcDir: etc, + Unit: apply.HostServerUnit, + UnitDir: filepath.Dir(discover.HostServerUnitPath), + User: apply.HostServerUser, + DryRun: *dryRun, + Log: func(format string, a ...any) { fmt.Printf(" "+format+"\n", a...) }, + } + p, err := cluster.BuildPlan(cfg) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + return 1 + } + fmt.Print(p.String()) + + if *dryRun { + return 0 + } + if !*yes { + fmt.Fprintln(os.Stderr, "\nNothing has happened yet. Pass --yes to carry this out, or --dry-run to see the files.") + return 1 + } + if os.Geteuid() != 0 { + fmt.Fprintln(os.Stderr, "\nthis writes under the server's directory and /etc/systemd: run it as root") + return 1 + } + + fmt.Println("\nJoining:") + if err := cluster.Apply(context.Background(), p); err != nil { + fmt.Fprintln(os.Stderr, "\nstopped: "+err.Error()) + return 1 + } + if st, err := state.Load(); err == nil { + st.Clustered, st.ClusterRole = true, m.Cluster.Role + if err := st.Save(); err != nil { + fmt.Fprintf(os.Stderr, "could not record cluster membership in %s: %v\n", state.Path, err) + } + } + fmt.Printf("\nDone. %s is in the cluster as %s.\n", cfg.Machine, roleWordCLI(m.Cluster.Role)) + return 0 +} + +func roleWordCLI(role string) string { + if role == "" { + return "the node that carries every task (no role)" + } + return "role " + role +} + // clusterDoctor probes the data-layer endpoints the operator names, so a node // can be checked before it is pointed at a cluster. It touches nothing. func clusterDoctor(args []string) int { diff --git a/internal/cluster/join.go b/internal/cluster/join.go new file mode 100644 index 0000000..2ab221f --- /dev/null +++ b/internal/cluster/join.go @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cluster + +// Joining a node to a cluster is, on the node, a small and mechanical thing: +// point the server at the shared store instead of its built-in one, put the +// secrets every node reads from disk in place, say which role it takes and +// what it must start after, and restart. This file is that, and nothing more. +// +// It is deliberately the node's half only. The shared registry -- which blob +// store, which coordinator, which roles exist -- is set up once for the whole +// cluster, not by every node that joins; a node that joins an existing cluster +// reads all of that from the shared store the moment its config points there, +// needing only the secret files the registry's File references name. So this +// places those files and writes the store config, and leaves the registry to +// the step that creates the cluster. +// +// Everything it would do is worked out first as a Plan, which --dry-run prints +// and Apply carries out: what is shown is what runs. Nothing here reaches +// another machine; the file is copied across and run on each node by whoever +// owns it, exactly as the rest of the installer works. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology" +) + +// Config is what a join needs to know about this machine: the cluster block +// from the topology, and where this installation keeps the server's files. +type Config struct { + Node topology.ClusterNode // this node's cluster block + Machine string // the machine's name, used as the hostname when the block gives none + EtcDir string // where config.json and the secret files live (the server's home) + Unit string // the server's systemd unit, e.g. "inbuxa-server.service" + UnitDir string // where units live, so drop-ins go in /.d + User string // the server's user, for ownership of what is written + DryRun bool // + Run func(ctx context.Context, name string, args ...string) error // exec; nil uses the real one + Log func(format string, a ...any) // progress; nil is silent +} + +// write is one file the join would put in place, rendered up front so a dry +// run shows its exact contents. +type write struct { + Path string + Mode os.FileMode + Desc string + Content string // for a plain write + From string // for an install from a source file; Content is then empty + Secret bool // a secret file: a dry run names it but does not show it +} + +// Plan is everything a join would do on this node, in the order it would do +// it, without doing any of it. +type Plan struct { + Config Config + Writes []write + Backup string // the existing config.json copied aside first, for rollback; "" if none + Restart bool // the server is restarted at the end + Role string // the role the node takes, "" for the node that carries everything +} + +// defaultRun runs a command and returns its combined output on failure. +func defaultRun(ctx context.Context, name string, args ...string) error { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + if msg := strings.TrimSpace(string(out)); msg != "" { + return fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, msg) + } + return fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) + } + return nil +} + +func (c Config) run() func(ctx context.Context, name string, args ...string) error { + if c.Run != nil { + return c.Run + } + return defaultRun +} + +func (c Config) log(format string, a ...any) { + if c.Log != nil { + c.Log(format, a...) + } +} + +func (c Config) configPath() string { return filepath.Join(c.EtcDir, "config.json") } +func (c Config) dropInDir() string { return filepath.Join(c.UnitDir, c.Unit+".d") } +func (c Config) secretPath(f string) string { + // A secret_file may be a bare name (kept beside config.json) or an + // absolute path the operator chose; honour both. + if filepath.IsAbs(f) { + return f + } + return filepath.Join(c.EtcDir, f) +} + +// hostname is what the node calls itself: the block's, or the machine name. +func (c Config) hostname() string { + if c.Node.Hostname != "" { + return c.Node.Hostname + } + return c.Machine +} + +// BuildPlan works out the join without touching the machine. It fails only on +// what it can tell is wrong from here -- a secret with no source that is not +// already in place -- leaving the addresses themselves for the server to judge +// when it starts. +func BuildPlan(c Config) (*Plan, error) { + if c.EtcDir == "" || c.Unit == "" || c.UnitDir == "" { + return nil, fmt.Errorf("a join needs the server's directory, unit and unit directory") + } + if _, err := os.Stat(c.EtcDir); err != nil { + return nil, fmt.Errorf("the server's directory %s is not there; a node joins a cluster after the server is installed on it", c.EtcDir) + } + p := &Plan{Config: c, Role: c.Node.Role, Restart: true} + + // The secrets every node reads from disk. The store's, then the + // coordinator's and the blob store's when the block names them. A secret + // with a source is installed from it; one without must already be there. + type sec struct{ file, from, what string } + secs := []sec{{c.Node.Store.SecretFile, c.Node.Store.SecretFrom, "the store password"}} + if cs := c.Node.Coordinator; cs != nil { + secs = append(secs, sec{cs.SecretFile, cs.SecretFrom, "the coordinator secret"}) + } + if b := c.Node.BlobStore; b != nil { + secs = append(secs, sec{b.SecretFile, b.SecretFrom, "the blob store secret"}) + } + for _, s := range secs { + dst := c.secretPath(s.file) + if s.from == "" { + if _, err := os.Stat(dst); err != nil { + return nil, fmt.Errorf("%s is not at %s and no secret_from says where to install it from", s.what, dst) + } + continue // already in place, left as it is + } + if _, err := os.Stat(s.from); err != nil { + return nil, fmt.Errorf("%s: the source %s is not there", s.what, s.from) + } + p.Writes = append(p.Writes, write{Path: dst, Mode: 0o400, Desc: s.what, From: s.from, Secret: true}) + } + + // The existing config.json, kept aside once so a node can be put back on + // its old store. Only when there is one and no copy has been made before. + if _, err := os.Stat(c.configPath()); err == nil { + backup := c.configPath() + ".prev" + if _, err := os.Stat(backup); err != nil { + p.Backup = backup + } + } + + cfg, err := storeConfig(c.Node.Store, c.secretPath(c.Node.Store.SecretFile)) + if err != nil { + return nil, err + } + p.Writes = append(p.Writes, write{Path: c.configPath(), Mode: 0o600, Desc: "the shared store this node reads and writes", Content: cfg}) + + p.Writes = append(p.Writes, write{ + Path: filepath.Join(c.dropInDir(), "cluster.conf"), Mode: 0o644, + Desc: "the node's hostname" + roleWord(c.Node.Role), Content: clusterDropIn(c.hostname(), c.Node.Role), + }) + if len(c.Node.After) > 0 { + p.Writes = append(p.Writes, write{ + Path: filepath.Join(c.dropInDir(), "cluster-order.conf"), Mode: 0o644, + Desc: "start this node after " + strings.Join(c.Node.After, ", "), Content: orderDropIn(c.Node.After), + }) + } + return p, nil +} + +// storeConfig is the server's config.json for a shared SQL store: the one +// store kind more than one node can share. The field names are the server's +// SQL-store config; the type it is given is the server's own store @type. +func storeConfig(s topology.ClusterStore, secretPath string) (string, error) { + pool := s.PoolMax + if pool == 0 { + pool = 40 + } + cfg := map[string]any{ + "@type": s.Type, + "host": s.Host, + "port": s.Port, + "database": s.Database, + "authUsername": s.User, + "authSecret": map[string]any{"@type": "File", "filePath": secretPath}, + "useTls": s.TLS, + "poolMaxConnections": pool, + } + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return "", err + } + return string(b) + "\n", nil +} + +func roleWord(role string) string { + if role == "" { + return "" + } + return " and role " + role +} + +// clusterDropIn sets the node's hostname, and its role unless it is the node +// that carries every task (no role). A drop-in, so the server's own +// environment file is left as it is. +func clusterDropIn(hostname, role string) string { + var b strings.Builder + b.WriteString("# Written by the inbuxa installer: this node's place in the cluster.\n") + b.WriteString("[Service]\n") + fmt.Fprintf(&b, "Environment=INBUXA_HOSTNAME=%s\n", hostname) + if role != "" { + fmt.Fprintf(&b, "Environment=INBUXA_ROLE=%s\n", role) + } + return b.String() +} + +// orderDropIn makes the node start after the units that bring up the path to +// its store, so it does not come up before the data layer it depends on. +func orderDropIn(after []string) string { + units := strings.Join(after, " ") + var b strings.Builder + b.WriteString("# Written by the inbuxa installer: reach the data layer before starting.\n") + b.WriteString("[Unit]\n") + fmt.Fprintf(&b, "After=%s\n", units) + fmt.Fprintf(&b, "Wants=%s\n", units) + return b.String() +} + +// Apply carries out the plan: the secrets, then the store config, then the +// drop-ins, then a reload and restart. The order matters -- the config and its +// secret are in place before the server is told to read them. +func Apply(ctx context.Context, p *Plan) error { + c := p.Config + run := c.run() + + if p.Backup != "" { + c.log("keeping the old store config at %s", p.Backup) + if !c.DryRun { + if err := copyFile(c.configPath(), p.Backup, 0o600); err != nil { + return fmt.Errorf("saving the old config aside: %w", err) + } + if c.User != "" { + chown(ctx, run, c.User, p.Backup) + } + } + } + + // Drop-in directory first, so the files below have somewhere to land. + if !c.DryRun { + if err := os.MkdirAll(c.dropInDir(), 0o755); err != nil { + return err + } + } + + for _, w := range p.Writes { + what := w.Path + if w.Secret { + what += " (secret, mode " + w.Mode.String() + ")" + } + c.log("writing %s -- %s", what, w.Desc) + if c.DryRun { + continue + } + if err := os.MkdirAll(filepath.Dir(w.Path), 0o750); err != nil { + return err + } + if w.From != "" { + if err := copyFile(w.From, w.Path, w.Mode); err != nil { + return fmt.Errorf("installing %s from %s: %w", w.Path, w.From, err) + } + } else if err := os.WriteFile(w.Path, []byte(w.Content), w.Mode); err != nil { + return err + } + // Secrets and the store config are the server's to read; the drop-ins + // are root's, as systemd wants them. + if c.User != "" && withinEtc(c, w.Path) { + chown(ctx, run, c.User, w.Path) + } + } + + if c.DryRun { + c.log("dry run: nothing was written, and the server was not restarted") + return nil + } + + c.log("reloading systemd and restarting %s", c.Unit) + if err := run(ctx, "systemctl", "daemon-reload"); err != nil { + return err + } + if p.Restart { + if err := run(ctx, "systemctl", "restart", c.Unit); err != nil { + return err + } + } + return nil +} + +// withinEtc is whether a path is one of the server's files (the store config +// or a secret beside it), as opposed to a systemd drop-in. +func withinEtc(c Config, path string) bool { + rel, err := filepath.Rel(c.EtcDir, path) + return err == nil && !strings.HasPrefix(rel, "..") +} + +func chown(ctx context.Context, run func(context.Context, string, ...string) error, user, path string) { + // Best effort: ownership needs root, and a dry run or a test without it + // should not fail over it. The mode is what keeps a secret unreadable. + _ = run(ctx, "chown", user+":"+user, path) +} + +func copyFile(src, dst string, mode os.FileMode) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + if err := os.WriteFile(dst, b, mode); err != nil { + return err + } + return os.Chmod(dst, mode) +} + +// String renders the plan the way --dry-run shows it, in the installer's +// house style. +func (p *Plan) String() string { + var b strings.Builder + c := p.Config + fmt.Fprintf(&b, "Join %s to the cluster as %s\n\n", c.hostname(), roleOrEverything(p.Role)) + n := 0 + step := func(title string, detail ...string) { + n++ + fmt.Fprintf(&b, "%2d. %s\n", n, title) + for _, d := range detail { + fmt.Fprintf(&b, " %s\n", d) + } + } + if p.Backup != "" { + step("Keep the current store config for rollback", p.Backup) + } + var secrets, files, dropins []write + for _, w := range p.Writes { + switch { + case w.Secret: + secrets = append(secrets, w) + case strings.HasSuffix(w.Path, ".conf"): + dropins = append(dropins, w) + default: + files = append(files, w) + } + } + if len(secrets) > 0 { + detail := make([]string, len(secrets)) + for i, w := range secrets { + detail[i] = fmt.Sprintf("%s, %s, from %s", w.Path, w.Mode, w.From) + } + step("Install the secrets this node reads from disk", detail...) + } + for _, w := range files { + step("Point the server at the shared store", append([]string{w.Path + ":"}, indent(w.Content)...)...) + } + for _, w := range dropins { + step("Configure the node's unit ("+filepath.Base(w.Path)+")", indent(w.Content)...) + } + if p.Restart { + step("Reload systemd and restart the server", c.Unit) + } + return b.String() +} + +func roleOrEverything(role string) string { + if role == "" { + return "the node that carries every task (no role)" + } + return "role " + role +} + +func indent(s string) []string { + var out []string + for _, line := range strings.Split(strings.TrimRight(s, "\n"), "\n") { + out = append(out, line) + } + return out +} diff --git a/internal/cluster/join_test.go b/internal/cluster/join_test.go new file mode 100644 index 0000000..5df1a26 --- /dev/null +++ b/internal/cluster/join_test.go @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cluster + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology" +) + +// recorder is a Run stub that remembers the commands Apply ran, so a test can +// check that it reloaded and restarted without touching the real systemd. +type recorder struct{ cmds []string } + +func (r *recorder) run(_ context.Context, name string, args ...string) error { + r.cmds = append(r.cmds, name+" "+strings.Join(args, " ")) + return nil +} + +// fixture lays out a server directory with a password source and an existing +// config.json, and returns a Config rooted entirely in a temp tree. +func fixture(t *testing.T, node topology.ClusterNode) (Config, *recorder) { + t.Helper() + root := t.TempDir() + etc := filepath.Join(root, "server") + if err := os.MkdirAll(etc, 0o750); err != nil { + t.Fatal(err) + } + // An existing single-node config, so a backup is made. + if err := os.WriteFile(filepath.Join(etc, "config.json"), []byte(`{"@type":"RocksDb"}`), 0o600); err != nil { + t.Fatal(err) + } + rec := &recorder{} + return Config{ + Node: node, + Machine: "mx2", + EtcDir: etc, + Unit: "inbuxa-server.service", + UnitDir: filepath.Join(root, "systemd"), + User: "", // no chown in tests: ownership needs root + Run: rec.run, + }, rec +} + +// clusterNode is a full cluster block with the password sources written to +// disk, for a happy-path test. +func clusterNode(t *testing.T, dir string) topology.ClusterNode { + t.Helper() + for name, body := range map[string]string{"pg.src": "pgpw", "nats.src": "natspw", "s3.src": "s3key"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + return topology.ClusterNode{ + Store: topology.ClusterStore{ + Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa", + SecretFile: "pg.pw", SecretFrom: filepath.Join(dir, "pg.src"), + }, + Coordinator: &topology.ClusterService{ + Addr: "10.0.0.2:4223", User: "inbuxa", + SecretFile: "nats.pw", SecretFrom: filepath.Join(dir, "nats.src"), + }, + BlobStore: &topology.ClusterBlob{ + Endpoint: "http://10.0.0.2:3910", Bucket: "blobs", + SecretFile: "s3.secret", SecretFrom: filepath.Join(dir, "s3.src"), + }, + Role: "peer", + After: []string{"haproxy.service", "wg.service"}, + } +} + +func TestBuildPlanAndApply(t *testing.T) { + src := t.TempDir() + node := clusterNode(t, src) + cfg, rec := fixture(t, node) + + p, err := BuildPlan(cfg) + if err != nil { + t.Fatal(err) + } + if p.Backup == "" { + t.Error("an existing config.json should be kept aside for rollback") + } + if err := Apply(context.Background(), p); err != nil { + t.Fatal(err) + } + + // The store config points at the shared store, with the server's own field + // names and a File secret at the installed path. + b, err := os.ReadFile(cfg.configPath()) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("config.json is not JSON: %v", err) + } + if got["@type"] != "PostgreSql" || got["host"] != "10.0.0.2" || got["poolMaxConnections"].(float64) != 40 { + t.Errorf("store config wrong: %v", got) + } + as, _ := got["authSecret"].(map[string]any) + if as["@type"] != "File" || as["filePath"] != filepath.Join(cfg.EtcDir, "pg.pw") { + t.Errorf("authSecret should be a File at the installed path, got %v", as) + } + + // The three secrets are installed, each 0400. + for _, f := range []string{"pg.pw", "nats.pw", "s3.secret"} { + fi, err := os.Stat(filepath.Join(cfg.EtcDir, f)) + if err != nil { + t.Fatalf("secret %s not installed: %v", f, err) + } + if fi.Mode().Perm() != 0o400 { + t.Errorf("%s mode is %v, want 0400", f, fi.Mode().Perm()) + } + } + + // The rollback copy holds the old config. + if b, _ := os.ReadFile(p.Backup); !strings.Contains(string(b), "RocksDb") { + t.Error("the backup should hold the previous config") + } + + // The drop-ins carry the hostname, the role and the boot order. + dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf")) + if !strings.Contains(string(dropIn), "INBUXA_HOSTNAME=mx2") || !strings.Contains(string(dropIn), "INBUXA_ROLE=peer") { + t.Errorf("cluster.conf wrong:\n%s", dropIn) + } + order, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster-order.conf")) + if !strings.Contains(string(order), "After=haproxy.service wg.service") { + t.Errorf("cluster-order.conf wrong:\n%s", order) + } + + // It reloaded systemd and restarted the server, and nothing else. + want := []string{"systemctl daemon-reload", "systemctl restart inbuxa-server.service"} + if strings.Join(rec.cmds, "|") != strings.Join(want, "|") { + t.Errorf("commands run: %v, want %v", rec.cmds, want) + } +} + +func TestNoRoleOmitsRoleLine(t *testing.T) { + src := t.TempDir() + node := clusterNode(t, src) + node.Role = "" // the node that carries everything + cfg, _ := fixture(t, node) + p, err := BuildPlan(cfg) + if err != nil { + t.Fatal(err) + } + if err := Apply(context.Background(), p); err != nil { + t.Fatal(err) + } + dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf")) + if strings.Contains(string(dropIn), "INBUXA_ROLE") { + t.Errorf("a node with no role should get no INBUXA_ROLE line:\n%s", dropIn) + } +} + +func TestDryRunWritesNothing(t *testing.T) { + src := t.TempDir() + cfg, rec := fixture(t, clusterNode(t, src)) + cfg.DryRun = true + p, err := BuildPlan(cfg) + if err != nil { + t.Fatal(err) + } + if err := Apply(context.Background(), p); err != nil { + t.Fatal(err) + } + // config.json is untouched (still the RocksDb one), no secret was written, + // no drop-in directory, and no command ran. + if b, _ := os.ReadFile(cfg.configPath()); !strings.Contains(string(b), "RocksDb") { + t.Error("a dry run must not rewrite config.json") + } + if _, err := os.Stat(filepath.Join(cfg.EtcDir, "pg.pw")); err == nil { + t.Error("a dry run must not install secrets") + } + if _, err := os.Stat(cfg.dropInDir()); err == nil { + t.Error("a dry run must not create the drop-in directory") + } + if len(rec.cmds) != 0 { + t.Errorf("a dry run must run no commands, ran %v", rec.cmds) + } +} + +func TestMissingSecretSource(t *testing.T) { + cfg, _ := fixture(t, topology.ClusterNode{ + Store: topology.ClusterStore{ + Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa", + SecretFile: "pg.pw", // no SecretFrom and not on disk + }, + }) + if _, err := BuildPlan(cfg); err == nil || !strings.Contains(err.Error(), "secret_from") { + t.Errorf("a secret with no source and no file should fail, got %v", err) + } +} + +func TestSecretAlreadyInPlace(t *testing.T) { + cfg, _ := fixture(t, topology.ClusterNode{ + Store: topology.ClusterStore{ + Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa", + SecretFile: "pg.pw", // already there, no source needed + }, + }) + if err := os.WriteFile(filepath.Join(cfg.EtcDir, "pg.pw"), []byte("x"), 0o400); err != nil { + t.Fatal(err) + } + p, err := BuildPlan(cfg) + if err != nil { + t.Fatalf("a secret already in place should be fine: %v", err) + } + for _, w := range p.Writes { + if w.Secret { + t.Error("a secret already in place should not be re-installed") + } + } +} + +func TestNoServerDir(t *testing.T) { + cfg := Config{Node: topology.ClusterNode{}, EtcDir: filepath.Join(t.TempDir(), "absent"), Unit: "u", UnitDir: "d"} + if _, err := BuildPlan(cfg); err == nil { + t.Error("a join before the server is installed should fail") + } +} diff --git a/internal/state/state.go b/internal/state/state.go index 39349ea..9cbba66 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -43,6 +43,12 @@ type State struct { // before it are moved over by a run with --key. PassThrough bool `json:"passThrough,omitempty"` + // Clustered is whether `inbuxa cluster join` pointed this node's server at + // a shared store; ClusterRole is the role it was given, empty for the node + // that carries every task. + Clustered bool `json:"clustered,omitempty"` + ClusterRole string `json:"clusterRole,omitempty"` + // RemovedAt is when `inbuxa uninstall` took the programs away and kept // the data. The shapes stay, so `install` run again picks the data up // rather than trying a first boot on a server configured long ago. diff --git a/internal/topology/cluster_test.go b/internal/topology/cluster_test.go new file mode 100644 index 0000000..e18cd47 --- /dev/null +++ b/internal/topology/cluster_test.go @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package topology + +import ( + "strings" + "testing" +) + +func store() ClusterStore { + return ClusterStore{Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa", SecretFile: "pg.pw"} +} + +func serverMachine(name string, c *ClusterNode) Machine { + return Machine{Name: name, Components: []Component{{Kind: "server", Shape: "host"}}, Cluster: c} +} + +func TestVersionOneStillValid(t *testing.T) { + v1 := &Topology{Version: 1, Domain: "example.com", Machines: []Machine{serverMachine("a", nil)}} + if err := v1.Validate(); err != nil { + t.Errorf("a version 1 file with no cluster block should still be valid: %v", err) + } +} + +func TestClusterBlockNeedsVersionTwo(t *testing.T) { + v1 := &Topology{Version: 1, Domain: "example.com", Machines: []Machine{serverMachine("a", &ClusterNode{Store: store()})}} + if err := v1.Validate(); err == nil || !strings.Contains(err.Error(), "version 2") { + t.Errorf("a cluster block under version 1 should be refused, got %v", err) + } +} + +func TestClusterNodeMustRunServer(t *testing.T) { + m := Machine{Name: "fe", Components: []Component{{Kind: "webmail", Shape: "container"}}, Cluster: &ClusterNode{Store: store()}} + // A lone webmail machine also needs a server somewhere; add one so the + // failure under test is the cluster-on-a-non-server one. + tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{serverMachine("a", nil), m}} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "runs no mail server") { + t.Errorf("a cluster block on a machine with no server should be refused, got %v", err) + } +} + +func TestClusteredMultiServer(t *testing.T) { + ok := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{ + serverMachine("a", &ClusterNode{Store: store()}), + serverMachine("b", &ClusterNode{Store: store(), Role: "peer"}), + }} + if err := ok.Validate(); err != nil { + t.Errorf("two mail nodes that both carry a cluster block is a clustered install: %v", err) + } + + mixed := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{ + serverMachine("a", &ClusterNode{Store: store()}), + serverMachine("b", nil), // a second server with no cluster block + }} + if err := mixed.Validate(); err == nil || !strings.Contains(err.Error(), "on every mail node") { + t.Errorf("two servers with only one cluster block should be refused, got %v", err) + } +} + +func TestClusterStoreValidation(t *testing.T) { + bad := store() + bad.Port = 0 + tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{serverMachine("a", &ClusterNode{Store: bad})}} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "port") { + t.Errorf("a store with no port should be refused, got %v", err) + } +} + +func TestClusterServiceNeedsSecretFile(t *testing.T) { + tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{ + serverMachine("a", &ClusterNode{Store: store(), Coordinator: &ClusterService{Addr: "h:4223"}}), + }} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "coordinator needs a secret_file") { + t.Errorf("a coordinator with no secret_file should be refused, got %v", err) + } +} + +func TestVersionRange(t *testing.T) { + tp := &Topology{Version: 3, Domain: "example.com", Machines: []Machine{serverMachine("a", nil)}} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "versions 1 to 2") { + t.Errorf("a future version should be refused, got %v", err) + } +} diff --git a/internal/topology/topology.go b/internal/topology/topology.go index b7b1cb0..6d7c251 100644 --- a/internal/topology/topology.go +++ b/internal/topology/topology.go @@ -10,12 +10,14 @@ // the designer, whatever draws it, emits this; `inbuxa apply -f` on each // machine is the only thing with privileges. // -// It describes front ends scaled out across machines, not a clustered mail -// server: one server, as many webmails and consoles as there are machines to -// put them on. A second mail node needs a shared store and cluster -// configuration, which is a larger product; the file's shape leaves room for -// it -- components on machines, with relationships -- so that version adds a -// kind rather than inventing a new file. +// Version 1 described front ends scaled out across machines, not a clustered +// mail server: one server, as many webmails and consoles as there are machines +// to put them on. Version 2 adds the second case without a new file: a machine +// that runs the server may carry a cluster block, which says where this node's +// shared store, coordinator and blob store are and what role it takes. The +// addresses in it are the operator's; nothing in the file is specific to any +// one cluster. A file with no cluster block is still a version 1 file in every +// way that matters, and this installer reads both. package topology import ( @@ -41,10 +43,11 @@ type Topology struct { // recognises its own part of the file; it defaults to the hostname, which is // what most people will call it anyway. type Machine struct { - Name string `json:"name"` - Components []Component `json:"components"` - Proxy string `json:"proxy,omitempty"` // caddy, snippets, none - Dir string `json:"dir,omitempty"` + Name string `json:"name"` + Components []Component `json:"components"` + Proxy string `json:"proxy,omitempty"` // caddy, snippets, none + Dir string `json:"dir,omitempty"` + Cluster *ClusterNode `json:"cluster,omitempty"` // version 2: this node's place in a cluster } // Component is one program on one machine. @@ -53,7 +56,63 @@ type Component struct { Shape string `json:"shape"` // container, host } -const Version = 1 +// ClusterNode is a server machine's membership in a clustered install: the +// shared data store this node reads and writes, the coordinator and blob store +// every node reaches, the node's role, and what it must start after. Every +// address in it is the operator's; the installer bakes in none of its own. +// +// What a joining node needs on disk is a store it can reach and the secrets +// the shared registry's File references point at. The coordinator and blob +// store are named here so the file is complete and so their secrets are placed +// beside the store's; the registry that points the cluster at them is set up +// once, as its own step, not by every node that joins. +type ClusterNode struct { + Store ClusterStore `json:"store"` // this node's shared data store (required) + Coordinator *ClusterService `json:"coordinator,omitempty"` // the message coordinator every node reads + BlobStore *ClusterBlob `json:"blob_store,omitempty"` // the blob store every node reads + Role string `json:"role,omitempty"` // the node's cluster role; empty on the node that carries every task + Hostname string `json:"hostname,omitempty"` // what the node calls itself (INBUXA_HOSTNAME); the machine name by default + After []string `json:"after,omitempty"` // units this node must start after, so it reaches the store at boot +} + +// ClusterStore is the shared store a node points its server at, in place of +// the single-node built-in store. It is a SQL store: the one kind more than +// one server can share. +type ClusterStore struct { + Type string `json:"type"` // the server store type, e.g. "PostgreSql" + Host string `json:"host"` // where this node reaches the store + Port int `json:"port"` // + Database string `json:"database"` // + User string `json:"user"` // the database user + SecretFile string `json:"secret_file"` // the file the server reads the password from + SecretFrom string `json:"secret_from,omitempty"` // a source to install that file from, 0400; empty means it is already there + TLS bool `json:"tls,omitempty"` // + PoolMax int `json:"pool_max,omitempty"` // connections in the pool; a sensible default when zero +} + +// ClusterService is the coordinator (the message bus nodes agree through): the +// address the registry names, and the secret each node reads from a local file. +type ClusterService struct { + Addr string `json:"addr"` // host:port + User string `json:"user,omitempty"` // + SecretFile string `json:"secret_file"` // the file the server reads the secret from + SecretFrom string `json:"secret_from,omitempty"` // a source to install it from, 0400; empty means already there +} + +// ClusterBlob is the blob store (where message bodies live): the endpoint the +// registry names, and the secret each node reads from a local file. +type ClusterBlob struct { + Endpoint string `json:"endpoint"` // the S3 endpoint, host:port or URL + Bucket string `json:"bucket,omitempty"` // + Region string `json:"region,omitempty"` // + SecretFile string `json:"secret_file"` // the file the server reads the secret key from + SecretFrom string `json:"secret_from,omitempty"` // a source to install it from, 0400; empty means already there +} + +// Clustered says whether this machine carries cluster configuration. +func (m Machine) Clustered() bool { return m.Cluster != nil } + +const Version = 2 var kinds = map[string]bool{"server": true, "console": true, "webmail": true} var shapes = map[string]bool{"container": true, "host": true} @@ -82,8 +141,8 @@ func Load(path string) (*Topology, error) { // spread over machines, and the interesting refusals are about arrangements // that cannot work rather than about syntax. func (t *Topology) Validate() error { - if t.Version != Version { - return fmt.Errorf("version %d: this installer writes and reads version %d", t.Version, Version) + if t.Version < 1 || t.Version > Version { + return fmt.Errorf("version %d: this installer reads versions 1 to %d", t.Version, Version) } if t.Domain == "" { return fmt.Errorf("no domain") @@ -94,7 +153,7 @@ func (t *Topology) Validate() error { return fmt.Errorf("no machines") } seen := map[string]bool{} - servers := 0 + servers, clustered := 0, 0 for _, m := range t.Machines { if m.Name == "" { return fmt.Errorf("a machine has no name") @@ -125,14 +184,70 @@ func (t *Topology) Validate() error { servers++ } } + if m.Cluster != nil { + if t.Version < 2 { + return fmt.Errorf("machine %q has a cluster block, which needs version 2", m.Name) + } + if !kindsHere["server"] { + return fmt.Errorf("machine %q has a cluster block but runs no mail server; only a server joins a cluster", m.Name) + } + if err := m.Cluster.validate(m.Name); err != nil { + return err + } + clustered++ + } } - switch servers { - case 1: - case 0: + switch { + case servers == 0: return fmt.Errorf("no machine runs the mail server") + case servers == 1: + // One server, clustered or not, is always allowed. + case clustered == servers: + // Every mail node carries cluster configuration: a clustered install. default: - return fmt.Errorf("%d machines run the mail server: one mail server for now, because a second "+ - "node needs a shared store and cluster configuration, which this installer does not set up", servers) + return fmt.Errorf("%d machines run the mail server but %d carry a cluster block: "+ + "a clustered install needs one on every mail node, or run a single server", servers, clustered) + } + return nil +} + +// validate checks a cluster node describes a store the installer can point a +// server at and secrets it can place. The addresses themselves are the +// operator's to get right; what it refuses is a block that cannot be acted on. +func (c *ClusterNode) validate(machine string) error { + fail := func(f string, a ...any) error { + return fmt.Errorf("machine %q cluster: %s", machine, fmt.Sprintf(f, a...)) + } + s := c.Store + switch { + case s.Type == "": + return fail("the store needs a type, e.g. \"PostgreSql\"") + case s.Host == "": + return fail("the store needs a host") + case s.Port <= 0 || s.Port > 65535: + return fail("the store port %d is not a port", s.Port) + case s.Database == "": + return fail("the store needs a database name") + case s.User == "": + return fail("the store needs a user") + case s.SecretFile == "": + return fail("the store needs a secret_file: where the server reads the password") + } + if c.Coordinator != nil { + if c.Coordinator.Addr == "" { + return fail("the coordinator needs an addr") + } + if c.Coordinator.SecretFile == "" { + return fail("the coordinator needs a secret_file") + } + } + if c.BlobStore != nil { + if c.BlobStore.Endpoint == "" { + return fail("the blob store needs an endpoint") + } + if c.BlobStore.SecretFile == "" { + return fail("the blob store needs a secret_file") + } } return nil } -- 2.54.0