diff --git a/cmd/inbuxa/main.go b/cmd/inbuxa/main.go index 5fc3f19..e73fb26 100644 --- a/cmd/inbuxa/main.go +++ b/cmd/inbuxa/main.go @@ -67,6 +67,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine inbuxa cluster doctor [flags] whether this node can reach a cluster's data layer (PostgreSQL, NATS, etcd, Patroni, S3) + inbuxa cluster join -f FILE + point this node's server at the shared store in + the file's cluster block and give it its role inbuxa version this program's version install flags: @@ -144,6 +147,15 @@ checked, so it fits any deployment; omit them all and it does nothing): --s3 HOST:PORT the S3 blob store, checked at /health --timeout DURATION how long each probe waits; default 5s +cluster join flags: + -f FILE the version 2 topology file; this machine's + cluster block says where its store, coordinator + and blob store are and what role it takes + --machine NAME which machine in the file this is; default: the + hostname + --dry-run print what it would write and stop + --yes do not ask for confirmation + deps flags: --server/--console/--webmail SHAPE the shapes to work out the needs for --install do it, rather than only saying what it would do @@ -1060,20 +1072,121 @@ func relayCmd(args []string) int { func clusterCmd(args []string) int { if len(args) == 0 { - fmt.Fprint(os.Stderr, "inbuxa cluster doctor [flags]\n\n"+usage) + fmt.Fprint(os.Stderr, "inbuxa cluster doctor|join [flags]\n\n"+usage) return 2 } switch args[0] { case "doctor": return clusterDoctor(args[1:]) + case "join": + return clusterJoin(args[1:]) case "-h", "--help", "help": fmt.Print(usage) return 0 } - fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor\n", args[0]) + fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor, join\n", args[0]) return 2 } +// clusterJoin points this node's server at the shared store named in the +// topology file's cluster block for this machine, places the secrets every +// node reads from disk, gives the node its role and boot order, and restarts +// it. It never reaches another machine: the file is copied across and run on +// each node, as the rest of the installer works. The registry that names the +// blob store and coordinator is set up once for the cluster, not here. +func clusterJoin(args []string) int { + fs := flag.NewFlagSet("cluster join", flag.ContinueOnError) + fs.Usage = func() { fmt.Print(usage) } + var ( + file = fs.String("f", "", "") + machineName = fs.String("machine", "", "") + dryRun = fs.Bool("dry-run", false, "") + yes = fs.Bool("yes", false, "") + ) + if err := fs.Parse(args); err != nil { + return 2 + } + if *file == "" { + fmt.Fprintln(os.Stderr, "which file? pass -f topology.json") + return 2 + } + t, err := topology.Load(*file) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + return 1 + } + name := *machineName + if name == "" { + name, _ = os.Hostname() + } + m, ok := t.Machine(name) + if !ok { + fmt.Fprintf(os.Stderr, "this machine is %q, which the file does not mention.\n", name) + fmt.Fprintf(os.Stderr, "It describes: %s\n", strings.Join(machineNames(t), ", ")) + fmt.Fprintln(os.Stderr, "Pass --machine to say which one this is.") + return 1 + } + if !m.Clustered() { + fmt.Fprintf(os.Stderr, "machine %q has no cluster block in this file; there is nothing to join.\n", name) + return 1 + } + if m.Shape("server") != "host" { + fmt.Fprintf(os.Stderr, "a cluster node runs the mail server on the host; machine %q runs it as %q.\n", name, m.Shape("server")) + return 1 + } + + etc := filepath.Join(m.Dir, "server") + cfg := cluster.Config{ + Node: *m.Cluster, + Machine: name, + EtcDir: etc, + Unit: apply.HostServerUnit, + UnitDir: filepath.Dir(discover.HostServerUnitPath), + User: apply.HostServerUser, + DryRun: *dryRun, + Log: func(format string, a ...any) { fmt.Printf(" "+format+"\n", a...) }, + } + p, err := cluster.BuildPlan(cfg) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + return 1 + } + fmt.Print(p.String()) + + if *dryRun { + return 0 + } + if !*yes { + fmt.Fprintln(os.Stderr, "\nNothing has happened yet. Pass --yes to carry this out, or --dry-run to see the files.") + return 1 + } + if os.Geteuid() != 0 { + fmt.Fprintln(os.Stderr, "\nthis writes under the server's directory and /etc/systemd: run it as root") + return 1 + } + + fmt.Println("\nJoining:") + if err := cluster.Apply(context.Background(), p); err != nil { + fmt.Fprintln(os.Stderr, "\nstopped: "+err.Error()) + return 1 + } + if st, err := state.Load(); err == nil { + st.Clustered, st.ClusterRole = true, m.Cluster.Role + if err := st.Save(); err != nil { + fmt.Fprintf(os.Stderr, "could not record cluster membership in %s: %v\n", state.Path, err) + } + } + fmt.Printf("\nDone. %s is in the cluster as %s.\n", cfg.Machine, roleWordCLI(m.Cluster.Role)) + return 0 +} + +func roleWordCLI(role string) string { + if role == "" { + return "the node that carries every task (no role)" + } + return "role " + role +} + // clusterDoctor probes the data-layer endpoints the operator names, so a node // can be checked before it is pointed at a cluster. It touches nothing. func clusterDoctor(args []string) int { diff --git a/internal/cluster/join.go b/internal/cluster/join.go new file mode 100644 index 0000000..2ab221f --- /dev/null +++ b/internal/cluster/join.go @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cluster + +// Joining a node to a cluster is, on the node, a small and mechanical thing: +// point the server at the shared store instead of its built-in one, put the +// secrets every node reads from disk in place, say which role it takes and +// what it must start after, and restart. This file is that, and nothing more. +// +// It is deliberately the node's half only. The shared registry -- which blob +// store, which coordinator, which roles exist -- is set up once for the whole +// cluster, not by every node that joins; a node that joins an existing cluster +// reads all of that from the shared store the moment its config points there, +// needing only the secret files the registry's File references name. So this +// places those files and writes the store config, and leaves the registry to +// the step that creates the cluster. +// +// Everything it would do is worked out first as a Plan, which --dry-run prints +// and Apply carries out: what is shown is what runs. Nothing here reaches +// another machine; the file is copied across and run on each node by whoever +// owns it, exactly as the rest of the installer works. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology" +) + +// Config is what a join needs to know about this machine: the cluster block +// from the topology, and where this installation keeps the server's files. +type Config struct { + Node topology.ClusterNode // this node's cluster block + Machine string // the machine's name, used as the hostname when the block gives none + EtcDir string // where config.json and the secret files live (the server's home) + Unit string // the server's systemd unit, e.g. "inbuxa-server.service" + UnitDir string // where units live, so drop-ins go in /.d + User string // the server's user, for ownership of what is written + DryRun bool // + Run func(ctx context.Context, name string, args ...string) error // exec; nil uses the real one + Log func(format string, a ...any) // progress; nil is silent +} + +// write is one file the join would put in place, rendered up front so a dry +// run shows its exact contents. +type write struct { + Path string + Mode os.FileMode + Desc string + Content string // for a plain write + From string // for an install from a source file; Content is then empty + Secret bool // a secret file: a dry run names it but does not show it +} + +// Plan is everything a join would do on this node, in the order it would do +// it, without doing any of it. +type Plan struct { + Config Config + Writes []write + Backup string // the existing config.json copied aside first, for rollback; "" if none + Restart bool // the server is restarted at the end + Role string // the role the node takes, "" for the node that carries everything +} + +// defaultRun runs a command and returns its combined output on failure. +func defaultRun(ctx context.Context, name string, args ...string) error { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + if msg := strings.TrimSpace(string(out)); msg != "" { + return fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, msg) + } + return fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) + } + return nil +} + +func (c Config) run() func(ctx context.Context, name string, args ...string) error { + if c.Run != nil { + return c.Run + } + return defaultRun +} + +func (c Config) log(format string, a ...any) { + if c.Log != nil { + c.Log(format, a...) + } +} + +func (c Config) configPath() string { return filepath.Join(c.EtcDir, "config.json") } +func (c Config) dropInDir() string { return filepath.Join(c.UnitDir, c.Unit+".d") } +func (c Config) secretPath(f string) string { + // A secret_file may be a bare name (kept beside config.json) or an + // absolute path the operator chose; honour both. + if filepath.IsAbs(f) { + return f + } + return filepath.Join(c.EtcDir, f) +} + +// hostname is what the node calls itself: the block's, or the machine name. +func (c Config) hostname() string { + if c.Node.Hostname != "" { + return c.Node.Hostname + } + return c.Machine +} + +// BuildPlan works out the join without touching the machine. It fails only on +// what it can tell is wrong from here -- a secret with no source that is not +// already in place -- leaving the addresses themselves for the server to judge +// when it starts. +func BuildPlan(c Config) (*Plan, error) { + if c.EtcDir == "" || c.Unit == "" || c.UnitDir == "" { + return nil, fmt.Errorf("a join needs the server's directory, unit and unit directory") + } + if _, err := os.Stat(c.EtcDir); err != nil { + return nil, fmt.Errorf("the server's directory %s is not there; a node joins a cluster after the server is installed on it", c.EtcDir) + } + p := &Plan{Config: c, Role: c.Node.Role, Restart: true} + + // The secrets every node reads from disk. The store's, then the + // coordinator's and the blob store's when the block names them. A secret + // with a source is installed from it; one without must already be there. + type sec struct{ file, from, what string } + secs := []sec{{c.Node.Store.SecretFile, c.Node.Store.SecretFrom, "the store password"}} + if cs := c.Node.Coordinator; cs != nil { + secs = append(secs, sec{cs.SecretFile, cs.SecretFrom, "the coordinator secret"}) + } + if b := c.Node.BlobStore; b != nil { + secs = append(secs, sec{b.SecretFile, b.SecretFrom, "the blob store secret"}) + } + for _, s := range secs { + dst := c.secretPath(s.file) + if s.from == "" { + if _, err := os.Stat(dst); err != nil { + return nil, fmt.Errorf("%s is not at %s and no secret_from says where to install it from", s.what, dst) + } + continue // already in place, left as it is + } + if _, err := os.Stat(s.from); err != nil { + return nil, fmt.Errorf("%s: the source %s is not there", s.what, s.from) + } + p.Writes = append(p.Writes, write{Path: dst, Mode: 0o400, Desc: s.what, From: s.from, Secret: true}) + } + + // The existing config.json, kept aside once so a node can be put back on + // its old store. Only when there is one and no copy has been made before. + if _, err := os.Stat(c.configPath()); err == nil { + backup := c.configPath() + ".prev" + if _, err := os.Stat(backup); err != nil { + p.Backup = backup + } + } + + cfg, err := storeConfig(c.Node.Store, c.secretPath(c.Node.Store.SecretFile)) + if err != nil { + return nil, err + } + p.Writes = append(p.Writes, write{Path: c.configPath(), Mode: 0o600, Desc: "the shared store this node reads and writes", Content: cfg}) + + p.Writes = append(p.Writes, write{ + Path: filepath.Join(c.dropInDir(), "cluster.conf"), Mode: 0o644, + Desc: "the node's hostname" + roleWord(c.Node.Role), Content: clusterDropIn(c.hostname(), c.Node.Role), + }) + if len(c.Node.After) > 0 { + p.Writes = append(p.Writes, write{ + Path: filepath.Join(c.dropInDir(), "cluster-order.conf"), Mode: 0o644, + Desc: "start this node after " + strings.Join(c.Node.After, ", "), Content: orderDropIn(c.Node.After), + }) + } + return p, nil +} + +// storeConfig is the server's config.json for a shared SQL store: the one +// store kind more than one node can share. The field names are the server's +// SQL-store config; the type it is given is the server's own store @type. +func storeConfig(s topology.ClusterStore, secretPath string) (string, error) { + pool := s.PoolMax + if pool == 0 { + pool = 40 + } + cfg := map[string]any{ + "@type": s.Type, + "host": s.Host, + "port": s.Port, + "database": s.Database, + "authUsername": s.User, + "authSecret": map[string]any{"@type": "File", "filePath": secretPath}, + "useTls": s.TLS, + "poolMaxConnections": pool, + } + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return "", err + } + return string(b) + "\n", nil +} + +func roleWord(role string) string { + if role == "" { + return "" + } + return " and role " + role +} + +// clusterDropIn sets the node's hostname, and its role unless it is the node +// that carries every task (no role). A drop-in, so the server's own +// environment file is left as it is. +func clusterDropIn(hostname, role string) string { + var b strings.Builder + b.WriteString("# Written by the inbuxa installer: this node's place in the cluster.\n") + b.WriteString("[Service]\n") + fmt.Fprintf(&b, "Environment=INBUXA_HOSTNAME=%s\n", hostname) + if role != "" { + fmt.Fprintf(&b, "Environment=INBUXA_ROLE=%s\n", role) + } + return b.String() +} + +// orderDropIn makes the node start after the units that bring up the path to +// its store, so it does not come up before the data layer it depends on. +func orderDropIn(after []string) string { + units := strings.Join(after, " ") + var b strings.Builder + b.WriteString("# Written by the inbuxa installer: reach the data layer before starting.\n") + b.WriteString("[Unit]\n") + fmt.Fprintf(&b, "After=%s\n", units) + fmt.Fprintf(&b, "Wants=%s\n", units) + return b.String() +} + +// Apply carries out the plan: the secrets, then the store config, then the +// drop-ins, then a reload and restart. The order matters -- the config and its +// secret are in place before the server is told to read them. +func Apply(ctx context.Context, p *Plan) error { + c := p.Config + run := c.run() + + if p.Backup != "" { + c.log("keeping the old store config at %s", p.Backup) + if !c.DryRun { + if err := copyFile(c.configPath(), p.Backup, 0o600); err != nil { + return fmt.Errorf("saving the old config aside: %w", err) + } + if c.User != "" { + chown(ctx, run, c.User, p.Backup) + } + } + } + + // Drop-in directory first, so the files below have somewhere to land. + if !c.DryRun { + if err := os.MkdirAll(c.dropInDir(), 0o755); err != nil { + return err + } + } + + for _, w := range p.Writes { + what := w.Path + if w.Secret { + what += " (secret, mode " + w.Mode.String() + ")" + } + c.log("writing %s -- %s", what, w.Desc) + if c.DryRun { + continue + } + if err := os.MkdirAll(filepath.Dir(w.Path), 0o750); err != nil { + return err + } + if w.From != "" { + if err := copyFile(w.From, w.Path, w.Mode); err != nil { + return fmt.Errorf("installing %s from %s: %w", w.Path, w.From, err) + } + } else if err := os.WriteFile(w.Path, []byte(w.Content), w.Mode); err != nil { + return err + } + // Secrets and the store config are the server's to read; the drop-ins + // are root's, as systemd wants them. + if c.User != "" && withinEtc(c, w.Path) { + chown(ctx, run, c.User, w.Path) + } + } + + if c.DryRun { + c.log("dry run: nothing was written, and the server was not restarted") + return nil + } + + c.log("reloading systemd and restarting %s", c.Unit) + if err := run(ctx, "systemctl", "daemon-reload"); err != nil { + return err + } + if p.Restart { + if err := run(ctx, "systemctl", "restart", c.Unit); err != nil { + return err + } + } + return nil +} + +// withinEtc is whether a path is one of the server's files (the store config +// or a secret beside it), as opposed to a systemd drop-in. +func withinEtc(c Config, path string) bool { + rel, err := filepath.Rel(c.EtcDir, path) + return err == nil && !strings.HasPrefix(rel, "..") +} + +func chown(ctx context.Context, run func(context.Context, string, ...string) error, user, path string) { + // Best effort: ownership needs root, and a dry run or a test without it + // should not fail over it. The mode is what keeps a secret unreadable. + _ = run(ctx, "chown", user+":"+user, path) +} + +func copyFile(src, dst string, mode os.FileMode) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + if err := os.WriteFile(dst, b, mode); err != nil { + return err + } + return os.Chmod(dst, mode) +} + +// String renders the plan the way --dry-run shows it, in the installer's +// house style. +func (p *Plan) String() string { + var b strings.Builder + c := p.Config + fmt.Fprintf(&b, "Join %s to the cluster as %s\n\n", c.hostname(), roleOrEverything(p.Role)) + n := 0 + step := func(title string, detail ...string) { + n++ + fmt.Fprintf(&b, "%2d. %s\n", n, title) + for _, d := range detail { + fmt.Fprintf(&b, " %s\n", d) + } + } + if p.Backup != "" { + step("Keep the current store config for rollback", p.Backup) + } + var secrets, files, dropins []write + for _, w := range p.Writes { + switch { + case w.Secret: + secrets = append(secrets, w) + case strings.HasSuffix(w.Path, ".conf"): + dropins = append(dropins, w) + default: + files = append(files, w) + } + } + if len(secrets) > 0 { + detail := make([]string, len(secrets)) + for i, w := range secrets { + detail[i] = fmt.Sprintf("%s, %s, from %s", w.Path, w.Mode, w.From) + } + step("Install the secrets this node reads from disk", detail...) + } + for _, w := range files { + step("Point the server at the shared store", append([]string{w.Path + ":"}, indent(w.Content)...)...) + } + for _, w := range dropins { + step("Configure the node's unit ("+filepath.Base(w.Path)+")", indent(w.Content)...) + } + if p.Restart { + step("Reload systemd and restart the server", c.Unit) + } + return b.String() +} + +func roleOrEverything(role string) string { + if role == "" { + return "the node that carries every task (no role)" + } + return "role " + role +} + +func indent(s string) []string { + var out []string + for _, line := range strings.Split(strings.TrimRight(s, "\n"), "\n") { + out = append(out, line) + } + return out +} diff --git a/internal/cluster/join_test.go b/internal/cluster/join_test.go new file mode 100644 index 0000000..5df1a26 --- /dev/null +++ b/internal/cluster/join_test.go @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cluster + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology" +) + +// recorder is a Run stub that remembers the commands Apply ran, so a test can +// check that it reloaded and restarted without touching the real systemd. +type recorder struct{ cmds []string } + +func (r *recorder) run(_ context.Context, name string, args ...string) error { + r.cmds = append(r.cmds, name+" "+strings.Join(args, " ")) + return nil +} + +// fixture lays out a server directory with a password source and an existing +// config.json, and returns a Config rooted entirely in a temp tree. +func fixture(t *testing.T, node topology.ClusterNode) (Config, *recorder) { + t.Helper() + root := t.TempDir() + etc := filepath.Join(root, "server") + if err := os.MkdirAll(etc, 0o750); err != nil { + t.Fatal(err) + } + // An existing single-node config, so a backup is made. + if err := os.WriteFile(filepath.Join(etc, "config.json"), []byte(`{"@type":"RocksDb"}`), 0o600); err != nil { + t.Fatal(err) + } + rec := &recorder{} + return Config{ + Node: node, + Machine: "mx2", + EtcDir: etc, + Unit: "inbuxa-server.service", + UnitDir: filepath.Join(root, "systemd"), + User: "", // no chown in tests: ownership needs root + Run: rec.run, + }, rec +} + +// clusterNode is a full cluster block with the password sources written to +// disk, for a happy-path test. +func clusterNode(t *testing.T, dir string) topology.ClusterNode { + t.Helper() + for name, body := range map[string]string{"pg.src": "pgpw", "nats.src": "natspw", "s3.src": "s3key"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + return topology.ClusterNode{ + Store: topology.ClusterStore{ + Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa", + SecretFile: "pg.pw", SecretFrom: filepath.Join(dir, "pg.src"), + }, + Coordinator: &topology.ClusterService{ + Addr: "10.0.0.2:4223", User: "inbuxa", + SecretFile: "nats.pw", SecretFrom: filepath.Join(dir, "nats.src"), + }, + BlobStore: &topology.ClusterBlob{ + Endpoint: "http://10.0.0.2:3910", Bucket: "blobs", + SecretFile: "s3.secret", SecretFrom: filepath.Join(dir, "s3.src"), + }, + Role: "peer", + After: []string{"haproxy.service", "wg.service"}, + } +} + +func TestBuildPlanAndApply(t *testing.T) { + src := t.TempDir() + node := clusterNode(t, src) + cfg, rec := fixture(t, node) + + p, err := BuildPlan(cfg) + if err != nil { + t.Fatal(err) + } + if p.Backup == "" { + t.Error("an existing config.json should be kept aside for rollback") + } + if err := Apply(context.Background(), p); err != nil { + t.Fatal(err) + } + + // The store config points at the shared store, with the server's own field + // names and a File secret at the installed path. + b, err := os.ReadFile(cfg.configPath()) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("config.json is not JSON: %v", err) + } + if got["@type"] != "PostgreSql" || got["host"] != "10.0.0.2" || got["poolMaxConnections"].(float64) != 40 { + t.Errorf("store config wrong: %v", got) + } + as, _ := got["authSecret"].(map[string]any) + if as["@type"] != "File" || as["filePath"] != filepath.Join(cfg.EtcDir, "pg.pw") { + t.Errorf("authSecret should be a File at the installed path, got %v", as) + } + + // The three secrets are installed, each 0400. + for _, f := range []string{"pg.pw", "nats.pw", "s3.secret"} { + fi, err := os.Stat(filepath.Join(cfg.EtcDir, f)) + if err != nil { + t.Fatalf("secret %s not installed: %v", f, err) + } + if fi.Mode().Perm() != 0o400 { + t.Errorf("%s mode is %v, want 0400", f, fi.Mode().Perm()) + } + } + + // The rollback copy holds the old config. + if b, _ := os.ReadFile(p.Backup); !strings.Contains(string(b), "RocksDb") { + t.Error("the backup should hold the previous config") + } + + // The drop-ins carry the hostname, the role and the boot order. + dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf")) + if !strings.Contains(string(dropIn), "INBUXA_HOSTNAME=mx2") || !strings.Contains(string(dropIn), "INBUXA_ROLE=peer") { + t.Errorf("cluster.conf wrong:\n%s", dropIn) + } + order, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster-order.conf")) + if !strings.Contains(string(order), "After=haproxy.service wg.service") { + t.Errorf("cluster-order.conf wrong:\n%s", order) + } + + // It reloaded systemd and restarted the server, and nothing else. + want := []string{"systemctl daemon-reload", "systemctl restart inbuxa-server.service"} + if strings.Join(rec.cmds, "|") != strings.Join(want, "|") { + t.Errorf("commands run: %v, want %v", rec.cmds, want) + } +} + +func TestNoRoleOmitsRoleLine(t *testing.T) { + src := t.TempDir() + node := clusterNode(t, src) + node.Role = "" // the node that carries everything + cfg, _ := fixture(t, node) + p, err := BuildPlan(cfg) + if err != nil { + t.Fatal(err) + } + if err := Apply(context.Background(), p); err != nil { + t.Fatal(err) + } + dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf")) + if strings.Contains(string(dropIn), "INBUXA_ROLE") { + t.Errorf("a node with no role should get no INBUXA_ROLE line:\n%s", dropIn) + } +} + +func TestDryRunWritesNothing(t *testing.T) { + src := t.TempDir() + cfg, rec := fixture(t, clusterNode(t, src)) + cfg.DryRun = true + p, err := BuildPlan(cfg) + if err != nil { + t.Fatal(err) + } + if err := Apply(context.Background(), p); err != nil { + t.Fatal(err) + } + // config.json is untouched (still the RocksDb one), no secret was written, + // no drop-in directory, and no command ran. + if b, _ := os.ReadFile(cfg.configPath()); !strings.Contains(string(b), "RocksDb") { + t.Error("a dry run must not rewrite config.json") + } + if _, err := os.Stat(filepath.Join(cfg.EtcDir, "pg.pw")); err == nil { + t.Error("a dry run must not install secrets") + } + if _, err := os.Stat(cfg.dropInDir()); err == nil { + t.Error("a dry run must not create the drop-in directory") + } + if len(rec.cmds) != 0 { + t.Errorf("a dry run must run no commands, ran %v", rec.cmds) + } +} + +func TestMissingSecretSource(t *testing.T) { + cfg, _ := fixture(t, topology.ClusterNode{ + Store: topology.ClusterStore{ + Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa", + SecretFile: "pg.pw", // no SecretFrom and not on disk + }, + }) + if _, err := BuildPlan(cfg); err == nil || !strings.Contains(err.Error(), "secret_from") { + t.Errorf("a secret with no source and no file should fail, got %v", err) + } +} + +func TestSecretAlreadyInPlace(t *testing.T) { + cfg, _ := fixture(t, topology.ClusterNode{ + Store: topology.ClusterStore{ + Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa", + SecretFile: "pg.pw", // already there, no source needed + }, + }) + if err := os.WriteFile(filepath.Join(cfg.EtcDir, "pg.pw"), []byte("x"), 0o400); err != nil { + t.Fatal(err) + } + p, err := BuildPlan(cfg) + if err != nil { + t.Fatalf("a secret already in place should be fine: %v", err) + } + for _, w := range p.Writes { + if w.Secret { + t.Error("a secret already in place should not be re-installed") + } + } +} + +func TestNoServerDir(t *testing.T) { + cfg := Config{Node: topology.ClusterNode{}, EtcDir: filepath.Join(t.TempDir(), "absent"), Unit: "u", UnitDir: "d"} + if _, err := BuildPlan(cfg); err == nil { + t.Error("a join before the server is installed should fail") + } +} diff --git a/internal/state/state.go b/internal/state/state.go index 39349ea..9cbba66 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -43,6 +43,12 @@ type State struct { // before it are moved over by a run with --key. PassThrough bool `json:"passThrough,omitempty"` + // Clustered is whether `inbuxa cluster join` pointed this node's server at + // a shared store; ClusterRole is the role it was given, empty for the node + // that carries every task. + Clustered bool `json:"clustered,omitempty"` + ClusterRole string `json:"clusterRole,omitempty"` + // RemovedAt is when `inbuxa uninstall` took the programs away and kept // the data. The shapes stay, so `install` run again picks the data up // rather than trying a first boot on a server configured long ago. diff --git a/internal/topology/cluster_test.go b/internal/topology/cluster_test.go new file mode 100644 index 0000000..e18cd47 --- /dev/null +++ b/internal/topology/cluster_test.go @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package topology + +import ( + "strings" + "testing" +) + +func store() ClusterStore { + return ClusterStore{Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa", SecretFile: "pg.pw"} +} + +func serverMachine(name string, c *ClusterNode) Machine { + return Machine{Name: name, Components: []Component{{Kind: "server", Shape: "host"}}, Cluster: c} +} + +func TestVersionOneStillValid(t *testing.T) { + v1 := &Topology{Version: 1, Domain: "example.com", Machines: []Machine{serverMachine("a", nil)}} + if err := v1.Validate(); err != nil { + t.Errorf("a version 1 file with no cluster block should still be valid: %v", err) + } +} + +func TestClusterBlockNeedsVersionTwo(t *testing.T) { + v1 := &Topology{Version: 1, Domain: "example.com", Machines: []Machine{serverMachine("a", &ClusterNode{Store: store()})}} + if err := v1.Validate(); err == nil || !strings.Contains(err.Error(), "version 2") { + t.Errorf("a cluster block under version 1 should be refused, got %v", err) + } +} + +func TestClusterNodeMustRunServer(t *testing.T) { + m := Machine{Name: "fe", Components: []Component{{Kind: "webmail", Shape: "container"}}, Cluster: &ClusterNode{Store: store()}} + // A lone webmail machine also needs a server somewhere; add one so the + // failure under test is the cluster-on-a-non-server one. + tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{serverMachine("a", nil), m}} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "runs no mail server") { + t.Errorf("a cluster block on a machine with no server should be refused, got %v", err) + } +} + +func TestClusteredMultiServer(t *testing.T) { + ok := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{ + serverMachine("a", &ClusterNode{Store: store()}), + serverMachine("b", &ClusterNode{Store: store(), Role: "peer"}), + }} + if err := ok.Validate(); err != nil { + t.Errorf("two mail nodes that both carry a cluster block is a clustered install: %v", err) + } + + mixed := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{ + serverMachine("a", &ClusterNode{Store: store()}), + serverMachine("b", nil), // a second server with no cluster block + }} + if err := mixed.Validate(); err == nil || !strings.Contains(err.Error(), "on every mail node") { + t.Errorf("two servers with only one cluster block should be refused, got %v", err) + } +} + +func TestClusterStoreValidation(t *testing.T) { + bad := store() + bad.Port = 0 + tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{serverMachine("a", &ClusterNode{Store: bad})}} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "port") { + t.Errorf("a store with no port should be refused, got %v", err) + } +} + +func TestClusterServiceNeedsSecretFile(t *testing.T) { + tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{ + serverMachine("a", &ClusterNode{Store: store(), Coordinator: &ClusterService{Addr: "h:4223"}}), + }} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "coordinator needs a secret_file") { + t.Errorf("a coordinator with no secret_file should be refused, got %v", err) + } +} + +func TestVersionRange(t *testing.T) { + tp := &Topology{Version: 3, Domain: "example.com", Machines: []Machine{serverMachine("a", nil)}} + if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "versions 1 to 2") { + t.Errorf("a future version should be refused, got %v", err) + } +} diff --git a/internal/topology/topology.go b/internal/topology/topology.go index b7b1cb0..6d7c251 100644 --- a/internal/topology/topology.go +++ b/internal/topology/topology.go @@ -10,12 +10,14 @@ // the designer, whatever draws it, emits this; `inbuxa apply -f` on each // machine is the only thing with privileges. // -// It describes front ends scaled out across machines, not a clustered mail -// server: one server, as many webmails and consoles as there are machines to -// put them on. A second mail node needs a shared store and cluster -// configuration, which is a larger product; the file's shape leaves room for -// it -- components on machines, with relationships -- so that version adds a -// kind rather than inventing a new file. +// Version 1 described front ends scaled out across machines, not a clustered +// mail server: one server, as many webmails and consoles as there are machines +// to put them on. Version 2 adds the second case without a new file: a machine +// that runs the server may carry a cluster block, which says where this node's +// shared store, coordinator and blob store are and what role it takes. The +// addresses in it are the operator's; nothing in the file is specific to any +// one cluster. A file with no cluster block is still a version 1 file in every +// way that matters, and this installer reads both. package topology import ( @@ -41,10 +43,11 @@ type Topology struct { // recognises its own part of the file; it defaults to the hostname, which is // what most people will call it anyway. type Machine struct { - Name string `json:"name"` - Components []Component `json:"components"` - Proxy string `json:"proxy,omitempty"` // caddy, snippets, none - Dir string `json:"dir,omitempty"` + Name string `json:"name"` + Components []Component `json:"components"` + Proxy string `json:"proxy,omitempty"` // caddy, snippets, none + Dir string `json:"dir,omitempty"` + Cluster *ClusterNode `json:"cluster,omitempty"` // version 2: this node's place in a cluster } // Component is one program on one machine. @@ -53,7 +56,63 @@ type Component struct { Shape string `json:"shape"` // container, host } -const Version = 1 +// ClusterNode is a server machine's membership in a clustered install: the +// shared data store this node reads and writes, the coordinator and blob store +// every node reaches, the node's role, and what it must start after. Every +// address in it is the operator's; the installer bakes in none of its own. +// +// What a joining node needs on disk is a store it can reach and the secrets +// the shared registry's File references point at. The coordinator and blob +// store are named here so the file is complete and so their secrets are placed +// beside the store's; the registry that points the cluster at them is set up +// once, as its own step, not by every node that joins. +type ClusterNode struct { + Store ClusterStore `json:"store"` // this node's shared data store (required) + Coordinator *ClusterService `json:"coordinator,omitempty"` // the message coordinator every node reads + BlobStore *ClusterBlob `json:"blob_store,omitempty"` // the blob store every node reads + Role string `json:"role,omitempty"` // the node's cluster role; empty on the node that carries every task + Hostname string `json:"hostname,omitempty"` // what the node calls itself (INBUXA_HOSTNAME); the machine name by default + After []string `json:"after,omitempty"` // units this node must start after, so it reaches the store at boot +} + +// ClusterStore is the shared store a node points its server at, in place of +// the single-node built-in store. It is a SQL store: the one kind more than +// one server can share. +type ClusterStore struct { + Type string `json:"type"` // the server store type, e.g. "PostgreSql" + Host string `json:"host"` // where this node reaches the store + Port int `json:"port"` // + Database string `json:"database"` // + User string `json:"user"` // the database user + SecretFile string `json:"secret_file"` // the file the server reads the password from + SecretFrom string `json:"secret_from,omitempty"` // a source to install that file from, 0400; empty means it is already there + TLS bool `json:"tls,omitempty"` // + PoolMax int `json:"pool_max,omitempty"` // connections in the pool; a sensible default when zero +} + +// ClusterService is the coordinator (the message bus nodes agree through): the +// address the registry names, and the secret each node reads from a local file. +type ClusterService struct { + Addr string `json:"addr"` // host:port + User string `json:"user,omitempty"` // + SecretFile string `json:"secret_file"` // the file the server reads the secret from + SecretFrom string `json:"secret_from,omitempty"` // a source to install it from, 0400; empty means already there +} + +// ClusterBlob is the blob store (where message bodies live): the endpoint the +// registry names, and the secret each node reads from a local file. +type ClusterBlob struct { + Endpoint string `json:"endpoint"` // the S3 endpoint, host:port or URL + Bucket string `json:"bucket,omitempty"` // + Region string `json:"region,omitempty"` // + SecretFile string `json:"secret_file"` // the file the server reads the secret key from + SecretFrom string `json:"secret_from,omitempty"` // a source to install it from, 0400; empty means already there +} + +// Clustered says whether this machine carries cluster configuration. +func (m Machine) Clustered() bool { return m.Cluster != nil } + +const Version = 2 var kinds = map[string]bool{"server": true, "console": true, "webmail": true} var shapes = map[string]bool{"container": true, "host": true} @@ -82,8 +141,8 @@ func Load(path string) (*Topology, error) { // spread over machines, and the interesting refusals are about arrangements // that cannot work rather than about syntax. func (t *Topology) Validate() error { - if t.Version != Version { - return fmt.Errorf("version %d: this installer writes and reads version %d", t.Version, Version) + if t.Version < 1 || t.Version > Version { + return fmt.Errorf("version %d: this installer reads versions 1 to %d", t.Version, Version) } if t.Domain == "" { return fmt.Errorf("no domain") @@ -94,7 +153,7 @@ func (t *Topology) Validate() error { return fmt.Errorf("no machines") } seen := map[string]bool{} - servers := 0 + servers, clustered := 0, 0 for _, m := range t.Machines { if m.Name == "" { return fmt.Errorf("a machine has no name") @@ -125,14 +184,70 @@ func (t *Topology) Validate() error { servers++ } } + if m.Cluster != nil { + if t.Version < 2 { + return fmt.Errorf("machine %q has a cluster block, which needs version 2", m.Name) + } + if !kindsHere["server"] { + return fmt.Errorf("machine %q has a cluster block but runs no mail server; only a server joins a cluster", m.Name) + } + if err := m.Cluster.validate(m.Name); err != nil { + return err + } + clustered++ + } } - switch servers { - case 1: - case 0: + switch { + case servers == 0: return fmt.Errorf("no machine runs the mail server") + case servers == 1: + // One server, clustered or not, is always allowed. + case clustered == servers: + // Every mail node carries cluster configuration: a clustered install. default: - return fmt.Errorf("%d machines run the mail server: one mail server for now, because a second "+ - "node needs a shared store and cluster configuration, which this installer does not set up", servers) + return fmt.Errorf("%d machines run the mail server but %d carry a cluster block: "+ + "a clustered install needs one on every mail node, or run a single server", servers, clustered) + } + return nil +} + +// validate checks a cluster node describes a store the installer can point a +// server at and secrets it can place. The addresses themselves are the +// operator's to get right; what it refuses is a block that cannot be acted on. +func (c *ClusterNode) validate(machine string) error { + fail := func(f string, a ...any) error { + return fmt.Errorf("machine %q cluster: %s", machine, fmt.Sprintf(f, a...)) + } + s := c.Store + switch { + case s.Type == "": + return fail("the store needs a type, e.g. \"PostgreSql\"") + case s.Host == "": + return fail("the store needs a host") + case s.Port <= 0 || s.Port > 65535: + return fail("the store port %d is not a port", s.Port) + case s.Database == "": + return fail("the store needs a database name") + case s.User == "": + return fail("the store needs a user") + case s.SecretFile == "": + return fail("the store needs a secret_file: where the server reads the password") + } + if c.Coordinator != nil { + if c.Coordinator.Addr == "" { + return fail("the coordinator needs an addr") + } + if c.Coordinator.SecretFile == "" { + return fail("the coordinator needs a secret_file") + } + } + if c.BlobStore != nil { + if c.BlobStore.Endpoint == "" { + return fail("the blob store needs an endpoint") + } + if c.BlobStore.SecretFile == "" { + return fail("the blob store needs a secret_file") + } } return nil }