Add inbuxa cluster join: configure a node into a cluster (topology v2) #54

Merged
jcoffey-dev merged 1 commits from feat/cluster-join into main 2026-11-12 14:04:13 +00:00
6 changed files with 959 additions and 21 deletions

No files matched your search

+115 -2
View File
@@ -67,6 +67,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine
inbuxa cluster doctor [flags]
whether this node can reach a cluster's data
layer (PostgreSQL, NATS, etcd, Patroni, S3)
inbuxa cluster join -f FILE
point this node's server at the shared store in
the file's cluster block and give it its role
inbuxa version this program's version
install flags:
@@ -144,6 +147,15 @@ checked, so it fits any deployment; omit them all and it does nothing):
--s3 HOST:PORT the S3 blob store, checked at /health
--timeout DURATION how long each probe waits; default 5s
cluster join flags:
-f FILE the version 2 topology file; this machine's
cluster block says where its store, coordinator
and blob store are and what role it takes
--machine NAME which machine in the file this is; default: the
hostname
--dry-run print what it would write and stop
--yes do not ask for confirmation
deps flags:
--server/--console/--webmail SHAPE the shapes to work out the needs for
--install do it, rather than only saying what it would do
@@ -1060,20 +1072,121 @@ func relayCmd(args []string) int {
func clusterCmd(args []string) int {
if len(args) == 0 {
fmt.Fprint(os.Stderr, "inbuxa cluster doctor [flags]\n\n"+usage)
fmt.Fprint(os.Stderr, "inbuxa cluster doctor|join [flags]\n\n"+usage)
return 2
}
switch args[0] {
case "doctor":
return clusterDoctor(args[1:])
case "join":
return clusterJoin(args[1:])
case "-h", "--help", "help":
fmt.Print(usage)
return 0
}
fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor\n", args[0])
fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor, join\n", args[0])
return 2
}
// clusterJoin points this node's server at the shared store named in the
// topology file's cluster block for this machine, places the secrets every
// node reads from disk, gives the node its role and boot order, and restarts
// it. It never reaches another machine: the file is copied across and run on
// each node, as the rest of the installer works. The registry that names the
// blob store and coordinator is set up once for the cluster, not here.
func clusterJoin(args []string) int {
fs := flag.NewFlagSet("cluster join", flag.ContinueOnError)
fs.Usage = func() { fmt.Print(usage) }
var (
file = fs.String("f", "", "")
machineName = fs.String("machine", "", "")
dryRun = fs.Bool("dry-run", false, "")
yes = fs.Bool("yes", false, "")
)
if err := fs.Parse(args); err != nil {
return 2
}
if *file == "" {
fmt.Fprintln(os.Stderr, "which file? pass -f topology.json")
return 2
}
t, err := topology.Load(*file)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
return 1
}
name := *machineName
if name == "" {
name, _ = os.Hostname()
}
m, ok := t.Machine(name)
if !ok {
fmt.Fprintf(os.Stderr, "this machine is %q, which the file does not mention.\n", name)
fmt.Fprintf(os.Stderr, "It describes: %s\n", strings.Join(machineNames(t), ", "))
fmt.Fprintln(os.Stderr, "Pass --machine to say which one this is.")
return 1
}
if !m.Clustered() {
fmt.Fprintf(os.Stderr, "machine %q has no cluster block in this file; there is nothing to join.\n", name)
return 1
}
if m.Shape("server") != "host" {
fmt.Fprintf(os.Stderr, "a cluster node runs the mail server on the host; machine %q runs it as %q.\n", name, m.Shape("server"))
return 1
}
etc := filepath.Join(m.Dir, "server")
cfg := cluster.Config{
Node: *m.Cluster,
Machine: name,
EtcDir: etc,
Unit: apply.HostServerUnit,
UnitDir: filepath.Dir(discover.HostServerUnitPath),
User: apply.HostServerUser,
DryRun: *dryRun,
Log: func(format string, a ...any) { fmt.Printf(" "+format+"\n", a...) },
}
p, err := cluster.BuildPlan(cfg)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
return 1
}
fmt.Print(p.String())
if *dryRun {
return 0
}
if !*yes {
fmt.Fprintln(os.Stderr, "\nNothing has happened yet. Pass --yes to carry this out, or --dry-run to see the files.")
return 1
}
if os.Geteuid() != 0 {
fmt.Fprintln(os.Stderr, "\nthis writes under the server's directory and /etc/systemd: run it as root")
return 1
}
fmt.Println("\nJoining:")
if err := cluster.Apply(context.Background(), p); err != nil {
fmt.Fprintln(os.Stderr, "\nstopped: "+err.Error())
return 1
}
if st, err := state.Load(); err == nil {
st.Clustered, st.ClusterRole = true, m.Cluster.Role
if err := st.Save(); err != nil {
fmt.Fprintf(os.Stderr, "could not record cluster membership in %s: %v\n", state.Path, err)
}
}
fmt.Printf("\nDone. %s is in the cluster as %s.\n", cfg.Machine, roleWordCLI(m.Cluster.Role))
return 0
}
func roleWordCLI(role string) string {
if role == "" {
return "the node that carries every task (no role)"
}
return "role " + role
}
// clusterDoctor probes the data-layer endpoints the operator names, so a node
// can be checked before it is pointed at a cluster. It touches nothing.
func clusterDoctor(args []string) int {
+392
View File
@@ -0,0 +1,392 @@
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
// SPDX-License-Identifier: AGPL-3.0-or-later
package cluster
// Joining a node to a cluster is, on the node, a small and mechanical thing:
// point the server at the shared store instead of its built-in one, put the
// secrets every node reads from disk in place, say which role it takes and
// what it must start after, and restart. This file is that, and nothing more.
//
// It is deliberately the node's half only. The shared registry -- which blob
// store, which coordinator, which roles exist -- is set up once for the whole
// cluster, not by every node that joins; a node that joins an existing cluster
// reads all of that from the shared store the moment its config points there,
// needing only the secret files the registry's File references name. So this
// places those files and writes the store config, and leaves the registry to
// the step that creates the cluster.
//
// Everything it would do is worked out first as a Plan, which --dry-run prints
// and Apply carries out: what is shown is what runs. Nothing here reaches
// another machine; the file is copied across and run on each node by whoever
// owns it, exactly as the rest of the installer works.
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology"
)
// Config is what a join needs to know about this machine: the cluster block
// from the topology, and where this installation keeps the server's files.
type Config struct {
Node topology.ClusterNode // this node's cluster block
Machine string // the machine's name, used as the hostname when the block gives none
EtcDir string // where config.json and the secret files live (the server's home)
Unit string // the server's systemd unit, e.g. "inbuxa-server.service"
UnitDir string // where units live, so drop-ins go in <UnitDir>/<Unit>.d
User string // the server's user, for ownership of what is written
DryRun bool //
Run func(ctx context.Context, name string, args ...string) error // exec; nil uses the real one
Log func(format string, a ...any) // progress; nil is silent
}
// write is one file the join would put in place, rendered up front so a dry
// run shows its exact contents.
type write struct {
Path string
Mode os.FileMode
Desc string
Content string // for a plain write
From string // for an install from a source file; Content is then empty
Secret bool // a secret file: a dry run names it but does not show it
}
// Plan is everything a join would do on this node, in the order it would do
// it, without doing any of it.
type Plan struct {
Config Config
Writes []write
Backup string // the existing config.json copied aside first, for rollback; "" if none
Restart bool // the server is restarted at the end
Role string // the role the node takes, "" for the node that carries everything
}
// defaultRun runs a command and returns its combined output on failure.
func defaultRun(ctx context.Context, name string, args ...string) error {
out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
if err != nil {
if msg := strings.TrimSpace(string(out)); msg != "" {
return fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, msg)
}
return fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err)
}
return nil
}
func (c Config) run() func(ctx context.Context, name string, args ...string) error {
if c.Run != nil {
return c.Run
}
return defaultRun
}
func (c Config) log(format string, a ...any) {
if c.Log != nil {
c.Log(format, a...)
}
}
func (c Config) configPath() string { return filepath.Join(c.EtcDir, "config.json") }
func (c Config) dropInDir() string { return filepath.Join(c.UnitDir, c.Unit+".d") }
func (c Config) secretPath(f string) string {
// A secret_file may be a bare name (kept beside config.json) or an
// absolute path the operator chose; honour both.
if filepath.IsAbs(f) {
return f
}
return filepath.Join(c.EtcDir, f)
}
// hostname is what the node calls itself: the block's, or the machine name.
func (c Config) hostname() string {
if c.Node.Hostname != "" {
return c.Node.Hostname
}
return c.Machine
}
// BuildPlan works out the join without touching the machine. It fails only on
// what it can tell is wrong from here -- a secret with no source that is not
// already in place -- leaving the addresses themselves for the server to judge
// when it starts.
func BuildPlan(c Config) (*Plan, error) {
if c.EtcDir == "" || c.Unit == "" || c.UnitDir == "" {
return nil, fmt.Errorf("a join needs the server's directory, unit and unit directory")
}
if _, err := os.Stat(c.EtcDir); err != nil {
return nil, fmt.Errorf("the server's directory %s is not there; a node joins a cluster after the server is installed on it", c.EtcDir)
}
p := &Plan{Config: c, Role: c.Node.Role, Restart: true}
// The secrets every node reads from disk. The store's, then the
// coordinator's and the blob store's when the block names them. A secret
// with a source is installed from it; one without must already be there.
type sec struct{ file, from, what string }
secs := []sec{{c.Node.Store.SecretFile, c.Node.Store.SecretFrom, "the store password"}}
if cs := c.Node.Coordinator; cs != nil {
secs = append(secs, sec{cs.SecretFile, cs.SecretFrom, "the coordinator secret"})
}
if b := c.Node.BlobStore; b != nil {
secs = append(secs, sec{b.SecretFile, b.SecretFrom, "the blob store secret"})
}
for _, s := range secs {
dst := c.secretPath(s.file)
if s.from == "" {
if _, err := os.Stat(dst); err != nil {
return nil, fmt.Errorf("%s is not at %s and no secret_from says where to install it from", s.what, dst)
}
continue // already in place, left as it is
}
if _, err := os.Stat(s.from); err != nil {
return nil, fmt.Errorf("%s: the source %s is not there", s.what, s.from)
}
p.Writes = append(p.Writes, write{Path: dst, Mode: 0o400, Desc: s.what, From: s.from, Secret: true})
}
// The existing config.json, kept aside once so a node can be put back on
// its old store. Only when there is one and no copy has been made before.
if _, err := os.Stat(c.configPath()); err == nil {
backup := c.configPath() + ".prev"
if _, err := os.Stat(backup); err != nil {
p.Backup = backup
}
}
cfg, err := storeConfig(c.Node.Store, c.secretPath(c.Node.Store.SecretFile))
if err != nil {
return nil, err
}
p.Writes = append(p.Writes, write{Path: c.configPath(), Mode: 0o600, Desc: "the shared store this node reads and writes", Content: cfg})
p.Writes = append(p.Writes, write{
Path: filepath.Join(c.dropInDir(), "cluster.conf"), Mode: 0o644,
Desc: "the node's hostname" + roleWord(c.Node.Role), Content: clusterDropIn(c.hostname(), c.Node.Role),
})
if len(c.Node.After) > 0 {
p.Writes = append(p.Writes, write{
Path: filepath.Join(c.dropInDir(), "cluster-order.conf"), Mode: 0o644,
Desc: "start this node after " + strings.Join(c.Node.After, ", "), Content: orderDropIn(c.Node.After),
})
}
return p, nil
}
// storeConfig is the server's config.json for a shared SQL store: the one
// store kind more than one node can share. The field names are the server's
// SQL-store config; the type it is given is the server's own store @type.
func storeConfig(s topology.ClusterStore, secretPath string) (string, error) {
pool := s.PoolMax
if pool == 0 {
pool = 40
}
cfg := map[string]any{
"@type": s.Type,
"host": s.Host,
"port": s.Port,
"database": s.Database,
"authUsername": s.User,
"authSecret": map[string]any{"@type": "File", "filePath": secretPath},
"useTls": s.TLS,
"poolMaxConnections": pool,
}
b, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return "", err
}
return string(b) + "\n", nil
}
func roleWord(role string) string {
if role == "" {
return ""
}
return " and role " + role
}
// clusterDropIn sets the node's hostname, and its role unless it is the node
// that carries every task (no role). A drop-in, so the server's own
// environment file is left as it is.
func clusterDropIn(hostname, role string) string {
var b strings.Builder
b.WriteString("# Written by the inbuxa installer: this node's place in the cluster.\n")
b.WriteString("[Service]\n")
fmt.Fprintf(&b, "Environment=INBUXA_HOSTNAME=%s\n", hostname)
if role != "" {
fmt.Fprintf(&b, "Environment=INBUXA_ROLE=%s\n", role)
}
return b.String()
}
// orderDropIn makes the node start after the units that bring up the path to
// its store, so it does not come up before the data layer it depends on.
func orderDropIn(after []string) string {
units := strings.Join(after, " ")
var b strings.Builder
b.WriteString("# Written by the inbuxa installer: reach the data layer before starting.\n")
b.WriteString("[Unit]\n")
fmt.Fprintf(&b, "After=%s\n", units)
fmt.Fprintf(&b, "Wants=%s\n", units)
return b.String()
}
// Apply carries out the plan: the secrets, then the store config, then the
// drop-ins, then a reload and restart. The order matters -- the config and its
// secret are in place before the server is told to read them.
func Apply(ctx context.Context, p *Plan) error {
c := p.Config
run := c.run()
if p.Backup != "" {
c.log("keeping the old store config at %s", p.Backup)
if !c.DryRun {
if err := copyFile(c.configPath(), p.Backup, 0o600); err != nil {
return fmt.Errorf("saving the old config aside: %w", err)
}
if c.User != "" {
chown(ctx, run, c.User, p.Backup)
}
}
}
// Drop-in directory first, so the files below have somewhere to land.
if !c.DryRun {
if err := os.MkdirAll(c.dropInDir(), 0o755); err != nil {
return err
}
}
for _, w := range p.Writes {
what := w.Path
if w.Secret {
what += " (secret, mode " + w.Mode.String() + ")"
}
c.log("writing %s -- %s", what, w.Desc)
if c.DryRun {
continue
}
if err := os.MkdirAll(filepath.Dir(w.Path), 0o750); err != nil {
return err
}
if w.From != "" {
if err := copyFile(w.From, w.Path, w.Mode); err != nil {
return fmt.Errorf("installing %s from %s: %w", w.Path, w.From, err)
}
} else if err := os.WriteFile(w.Path, []byte(w.Content), w.Mode); err != nil {
return err
}
// Secrets and the store config are the server's to read; the drop-ins
// are root's, as systemd wants them.
if c.User != "" && withinEtc(c, w.Path) {
chown(ctx, run, c.User, w.Path)
}
}
if c.DryRun {
c.log("dry run: nothing was written, and the server was not restarted")
return nil
}
c.log("reloading systemd and restarting %s", c.Unit)
if err := run(ctx, "systemctl", "daemon-reload"); err != nil {
return err
}
if p.Restart {
if err := run(ctx, "systemctl", "restart", c.Unit); err != nil {
return err
}
}
return nil
}
// withinEtc is whether a path is one of the server's files (the store config
// or a secret beside it), as opposed to a systemd drop-in.
func withinEtc(c Config, path string) bool {
rel, err := filepath.Rel(c.EtcDir, path)
return err == nil && !strings.HasPrefix(rel, "..")
}
func chown(ctx context.Context, run func(context.Context, string, ...string) error, user, path string) {
// Best effort: ownership needs root, and a dry run or a test without it
// should not fail over it. The mode is what keeps a secret unreadable.
_ = run(ctx, "chown", user+":"+user, path)
}
func copyFile(src, dst string, mode os.FileMode) error {
b, err := os.ReadFile(src)
if err != nil {
return err
}
if err := os.WriteFile(dst, b, mode); err != nil {
return err
}
return os.Chmod(dst, mode)
}
// String renders the plan the way --dry-run shows it, in the installer's
// house style.
func (p *Plan) String() string {
var b strings.Builder
c := p.Config
fmt.Fprintf(&b, "Join %s to the cluster as %s\n\n", c.hostname(), roleOrEverything(p.Role))
n := 0
step := func(title string, detail ...string) {
n++
fmt.Fprintf(&b, "%2d. %s\n", n, title)
for _, d := range detail {
fmt.Fprintf(&b, " %s\n", d)
}
}
if p.Backup != "" {
step("Keep the current store config for rollback", p.Backup)
}
var secrets, files, dropins []write
for _, w := range p.Writes {
switch {
case w.Secret:
secrets = append(secrets, w)
case strings.HasSuffix(w.Path, ".conf"):
dropins = append(dropins, w)
default:
files = append(files, w)
}
}
if len(secrets) > 0 {
detail := make([]string, len(secrets))
for i, w := range secrets {
detail[i] = fmt.Sprintf("%s, %s, from %s", w.Path, w.Mode, w.From)
}
step("Install the secrets this node reads from disk", detail...)
}
for _, w := range files {
step("Point the server at the shared store", append([]string{w.Path + ":"}, indent(w.Content)...)...)
}
for _, w := range dropins {
step("Configure the node's unit ("+filepath.Base(w.Path)+")", indent(w.Content)...)
}
if p.Restart {
step("Reload systemd and restart the server", c.Unit)
}
return b.String()
}
func roleOrEverything(role string) string {
if role == "" {
return "the node that carries every task (no role)"
}
return "role " + role
}
func indent(s string) []string {
var out []string
for _, line := range strings.Split(strings.TrimRight(s, "\n"), "\n") {
out = append(out, line)
}
return out
}
+228
View File
@@ -0,0 +1,228 @@
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
// SPDX-License-Identifier: AGPL-3.0-or-later
package cluster
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology"
)
// recorder is a Run stub that remembers the commands Apply ran, so a test can
// check that it reloaded and restarted without touching the real systemd.
type recorder struct{ cmds []string }
func (r *recorder) run(_ context.Context, name string, args ...string) error {
r.cmds = append(r.cmds, name+" "+strings.Join(args, " "))
return nil
}
// fixture lays out a server directory with a password source and an existing
// config.json, and returns a Config rooted entirely in a temp tree.
func fixture(t *testing.T, node topology.ClusterNode) (Config, *recorder) {
t.Helper()
root := t.TempDir()
etc := filepath.Join(root, "server")
if err := os.MkdirAll(etc, 0o750); err != nil {
t.Fatal(err)
}
// An existing single-node config, so a backup is made.
if err := os.WriteFile(filepath.Join(etc, "config.json"), []byte(`{"@type":"RocksDb"}`), 0o600); err != nil {
t.Fatal(err)
}
rec := &recorder{}
return Config{
Node: node,
Machine: "mx2",
EtcDir: etc,
Unit: "inbuxa-server.service",
UnitDir: filepath.Join(root, "systemd"),
User: "", // no chown in tests: ownership needs root
Run: rec.run,
}, rec
}
// clusterNode is a full cluster block with the password sources written to
// disk, for a happy-path test.
func clusterNode(t *testing.T, dir string) topology.ClusterNode {
t.Helper()
for name, body := range map[string]string{"pg.src": "pgpw", "nats.src": "natspw", "s3.src": "s3key"} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
return topology.ClusterNode{
Store: topology.ClusterStore{
Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa",
SecretFile: "pg.pw", SecretFrom: filepath.Join(dir, "pg.src"),
},
Coordinator: &topology.ClusterService{
Addr: "10.0.0.2:4223", User: "inbuxa",
SecretFile: "nats.pw", SecretFrom: filepath.Join(dir, "nats.src"),
},
BlobStore: &topology.ClusterBlob{
Endpoint: "http://10.0.0.2:3910", Bucket: "blobs",
SecretFile: "s3.secret", SecretFrom: filepath.Join(dir, "s3.src"),
},
Role: "peer",
After: []string{"haproxy.service", "wg.service"},
}
}
func TestBuildPlanAndApply(t *testing.T) {
src := t.TempDir()
node := clusterNode(t, src)
cfg, rec := fixture(t, node)
p, err := BuildPlan(cfg)
if err != nil {
t.Fatal(err)
}
if p.Backup == "" {
t.Error("an existing config.json should be kept aside for rollback")
}
if err := Apply(context.Background(), p); err != nil {
t.Fatal(err)
}
// The store config points at the shared store, with the server's own field
// names and a File secret at the installed path.
b, err := os.ReadFile(cfg.configPath())
if err != nil {
t.Fatal(err)
}
var got map[string]any
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("config.json is not JSON: %v", err)
}
if got["@type"] != "PostgreSql" || got["host"] != "10.0.0.2" || got["poolMaxConnections"].(float64) != 40 {
t.Errorf("store config wrong: %v", got)
}
as, _ := got["authSecret"].(map[string]any)
if as["@type"] != "File" || as["filePath"] != filepath.Join(cfg.EtcDir, "pg.pw") {
t.Errorf("authSecret should be a File at the installed path, got %v", as)
}
// The three secrets are installed, each 0400.
for _, f := range []string{"pg.pw", "nats.pw", "s3.secret"} {
fi, err := os.Stat(filepath.Join(cfg.EtcDir, f))
if err != nil {
t.Fatalf("secret %s not installed: %v", f, err)
}
if fi.Mode().Perm() != 0o400 {
t.Errorf("%s mode is %v, want 0400", f, fi.Mode().Perm())
}
}
// The rollback copy holds the old config.
if b, _ := os.ReadFile(p.Backup); !strings.Contains(string(b), "RocksDb") {
t.Error("the backup should hold the previous config")
}
// The drop-ins carry the hostname, the role and the boot order.
dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf"))
if !strings.Contains(string(dropIn), "INBUXA_HOSTNAME=mx2") || !strings.Contains(string(dropIn), "INBUXA_ROLE=peer") {
t.Errorf("cluster.conf wrong:\n%s", dropIn)
}
order, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster-order.conf"))
if !strings.Contains(string(order), "After=haproxy.service wg.service") {
t.Errorf("cluster-order.conf wrong:\n%s", order)
}
// It reloaded systemd and restarted the server, and nothing else.
want := []string{"systemctl daemon-reload", "systemctl restart inbuxa-server.service"}
if strings.Join(rec.cmds, "|") != strings.Join(want, "|") {
t.Errorf("commands run: %v, want %v", rec.cmds, want)
}
}
func TestNoRoleOmitsRoleLine(t *testing.T) {
src := t.TempDir()
node := clusterNode(t, src)
node.Role = "" // the node that carries everything
cfg, _ := fixture(t, node)
p, err := BuildPlan(cfg)
if err != nil {
t.Fatal(err)
}
if err := Apply(context.Background(), p); err != nil {
t.Fatal(err)
}
dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf"))
if strings.Contains(string(dropIn), "INBUXA_ROLE") {
t.Errorf("a node with no role should get no INBUXA_ROLE line:\n%s", dropIn)
}
}
func TestDryRunWritesNothing(t *testing.T) {
src := t.TempDir()
cfg, rec := fixture(t, clusterNode(t, src))
cfg.DryRun = true
p, err := BuildPlan(cfg)
if err != nil {
t.Fatal(err)
}
if err := Apply(context.Background(), p); err != nil {
t.Fatal(err)
}
// config.json is untouched (still the RocksDb one), no secret was written,
// no drop-in directory, and no command ran.
if b, _ := os.ReadFile(cfg.configPath()); !strings.Contains(string(b), "RocksDb") {
t.Error("a dry run must not rewrite config.json")
}
if _, err := os.Stat(filepath.Join(cfg.EtcDir, "pg.pw")); err == nil {
t.Error("a dry run must not install secrets")
}
if _, err := os.Stat(cfg.dropInDir()); err == nil {
t.Error("a dry run must not create the drop-in directory")
}
if len(rec.cmds) != 0 {
t.Errorf("a dry run must run no commands, ran %v", rec.cmds)
}
}
func TestMissingSecretSource(t *testing.T) {
cfg, _ := fixture(t, topology.ClusterNode{
Store: topology.ClusterStore{
Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa",
SecretFile: "pg.pw", // no SecretFrom and not on disk
},
})
if _, err := BuildPlan(cfg); err == nil || !strings.Contains(err.Error(), "secret_from") {
t.Errorf("a secret with no source and no file should fail, got %v", err)
}
}
func TestSecretAlreadyInPlace(t *testing.T) {
cfg, _ := fixture(t, topology.ClusterNode{
Store: topology.ClusterStore{
Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa",
SecretFile: "pg.pw", // already there, no source needed
},
})
if err := os.WriteFile(filepath.Join(cfg.EtcDir, "pg.pw"), []byte("x"), 0o400); err != nil {
t.Fatal(err)
}
p, err := BuildPlan(cfg)
if err != nil {
t.Fatalf("a secret already in place should be fine: %v", err)
}
for _, w := range p.Writes {
if w.Secret {
t.Error("a secret already in place should not be re-installed")
}
}
}
func TestNoServerDir(t *testing.T) {
cfg := Config{Node: topology.ClusterNode{}, EtcDir: filepath.Join(t.TempDir(), "absent"), Unit: "u", UnitDir: "d"}
if _, err := BuildPlan(cfg); err == nil {
t.Error("a join before the server is installed should fail")
}
}
+6
View File
@@ -43,6 +43,12 @@ type State struct {
// before it are moved over by a run with --key.
PassThrough bool `json:"passThrough,omitempty"`
// Clustered is whether `inbuxa cluster join` pointed this node's server at
// a shared store; ClusterRole is the role it was given, empty for the node
// that carries every task.
Clustered bool `json:"clustered,omitempty"`
ClusterRole string `json:"clusterRole,omitempty"`
// RemovedAt is when `inbuxa uninstall` took the programs away and kept
// the data. The shapes stay, so `install` run again picks the data up
// rather than trying a first boot on a server configured long ago.
+84
View File
@@ -0,0 +1,84 @@
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
// SPDX-License-Identifier: AGPL-3.0-or-later
package topology
import (
"strings"
"testing"
)
func store() ClusterStore {
return ClusterStore{Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa", SecretFile: "pg.pw"}
}
func serverMachine(name string, c *ClusterNode) Machine {
return Machine{Name: name, Components: []Component{{Kind: "server", Shape: "host"}}, Cluster: c}
}
func TestVersionOneStillValid(t *testing.T) {
v1 := &Topology{Version: 1, Domain: "example.com", Machines: []Machine{serverMachine("a", nil)}}
if err := v1.Validate(); err != nil {
t.Errorf("a version 1 file with no cluster block should still be valid: %v", err)
}
}
func TestClusterBlockNeedsVersionTwo(t *testing.T) {
v1 := &Topology{Version: 1, Domain: "example.com", Machines: []Machine{serverMachine("a", &ClusterNode{Store: store()})}}
if err := v1.Validate(); err == nil || !strings.Contains(err.Error(), "version 2") {
t.Errorf("a cluster block under version 1 should be refused, got %v", err)
}
}
func TestClusterNodeMustRunServer(t *testing.T) {
m := Machine{Name: "fe", Components: []Component{{Kind: "webmail", Shape: "container"}}, Cluster: &ClusterNode{Store: store()}}
// A lone webmail machine also needs a server somewhere; add one so the
// failure under test is the cluster-on-a-non-server one.
tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{serverMachine("a", nil), m}}
if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "runs no mail server") {
t.Errorf("a cluster block on a machine with no server should be refused, got %v", err)
}
}
func TestClusteredMultiServer(t *testing.T) {
ok := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{
serverMachine("a", &ClusterNode{Store: store()}),
serverMachine("b", &ClusterNode{Store: store(), Role: "peer"}),
}}
if err := ok.Validate(); err != nil {
t.Errorf("two mail nodes that both carry a cluster block is a clustered install: %v", err)
}
mixed := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{
serverMachine("a", &ClusterNode{Store: store()}),
serverMachine("b", nil), // a second server with no cluster block
}}
if err := mixed.Validate(); err == nil || !strings.Contains(err.Error(), "on every mail node") {
t.Errorf("two servers with only one cluster block should be refused, got %v", err)
}
}
func TestClusterStoreValidation(t *testing.T) {
bad := store()
bad.Port = 0
tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{serverMachine("a", &ClusterNode{Store: bad})}}
if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "port") {
t.Errorf("a store with no port should be refused, got %v", err)
}
}
func TestClusterServiceNeedsSecretFile(t *testing.T) {
tp := &Topology{Version: 2, Domain: "example.com", Machines: []Machine{
serverMachine("a", &ClusterNode{Store: store(), Coordinator: &ClusterService{Addr: "h:4223"}}),
}}
if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "coordinator needs a secret_file") {
t.Errorf("a coordinator with no secret_file should be refused, got %v", err)
}
}
func TestVersionRange(t *testing.T) {
tp := &Topology{Version: 3, Domain: "example.com", Machines: []Machine{serverMachine("a", nil)}}
if err := tp.Validate(); err == nil || !strings.Contains(err.Error(), "versions 1 to 2") {
t.Errorf("a future version should be refused, got %v", err)
}
}
+134 -19
View File
@@ -10,12 +10,14 @@
// the designer, whatever draws it, emits this; `inbuxa apply -f` on each
// machine is the only thing with privileges.
//
// It describes front ends scaled out across machines, not a clustered mail
// server: one server, as many webmails and consoles as there are machines to
// put them on. A second mail node needs a shared store and cluster
// configuration, which is a larger product; the file's shape leaves room for
// it -- components on machines, with relationships -- so that version adds a
// kind rather than inventing a new file.
// Version 1 described front ends scaled out across machines, not a clustered
// mail server: one server, as many webmails and consoles as there are machines
// to put them on. Version 2 adds the second case without a new file: a machine
// that runs the server may carry a cluster block, which says where this node's
// shared store, coordinator and blob store are and what role it takes. The
// addresses in it are the operator's; nothing in the file is specific to any
// one cluster. A file with no cluster block is still a version 1 file in every
// way that matters, and this installer reads both.
package topology
import (
@@ -41,10 +43,11 @@ type Topology struct {
// recognises its own part of the file; it defaults to the hostname, which is
// what most people will call it anyway.
type Machine struct {
Name string `json:"name"`
Components []Component `json:"components"`
Proxy string `json:"proxy,omitempty"` // caddy, snippets, none
Dir string `json:"dir,omitempty"`
Name string `json:"name"`
Components []Component `json:"components"`
Proxy string `json:"proxy,omitempty"` // caddy, snippets, none
Dir string `json:"dir,omitempty"`
Cluster *ClusterNode `json:"cluster,omitempty"` // version 2: this node's place in a cluster
}
// Component is one program on one machine.
@@ -53,7 +56,63 @@ type Component struct {
Shape string `json:"shape"` // container, host
}
const Version = 1
// ClusterNode is a server machine's membership in a clustered install: the
// shared data store this node reads and writes, the coordinator and blob store
// every node reaches, the node's role, and what it must start after. Every
// address in it is the operator's; the installer bakes in none of its own.
//
// What a joining node needs on disk is a store it can reach and the secrets
// the shared registry's File references point at. The coordinator and blob
// store are named here so the file is complete and so their secrets are placed
// beside the store's; the registry that points the cluster at them is set up
// once, as its own step, not by every node that joins.
type ClusterNode struct {
Store ClusterStore `json:"store"` // this node's shared data store (required)
Coordinator *ClusterService `json:"coordinator,omitempty"` // the message coordinator every node reads
BlobStore *ClusterBlob `json:"blob_store,omitempty"` // the blob store every node reads
Role string `json:"role,omitempty"` // the node's cluster role; empty on the node that carries every task
Hostname string `json:"hostname,omitempty"` // what the node calls itself (INBUXA_HOSTNAME); the machine name by default
After []string `json:"after,omitempty"` // units this node must start after, so it reaches the store at boot
}
// ClusterStore is the shared store a node points its server at, in place of
// the single-node built-in store. It is a SQL store: the one kind more than
// one server can share.
type ClusterStore struct {
Type string `json:"type"` // the server store type, e.g. "PostgreSql"
Host string `json:"host"` // where this node reaches the store
Port int `json:"port"` //
Database string `json:"database"` //
User string `json:"user"` // the database user
SecretFile string `json:"secret_file"` // the file the server reads the password from
SecretFrom string `json:"secret_from,omitempty"` // a source to install that file from, 0400; empty means it is already there
TLS bool `json:"tls,omitempty"` //
PoolMax int `json:"pool_max,omitempty"` // connections in the pool; a sensible default when zero
}
// ClusterService is the coordinator (the message bus nodes agree through): the
// address the registry names, and the secret each node reads from a local file.
type ClusterService struct {
Addr string `json:"addr"` // host:port
User string `json:"user,omitempty"` //
SecretFile string `json:"secret_file"` // the file the server reads the secret from
SecretFrom string `json:"secret_from,omitempty"` // a source to install it from, 0400; empty means already there
}
// ClusterBlob is the blob store (where message bodies live): the endpoint the
// registry names, and the secret each node reads from a local file.
type ClusterBlob struct {
Endpoint string `json:"endpoint"` // the S3 endpoint, host:port or URL
Bucket string `json:"bucket,omitempty"` //
Region string `json:"region,omitempty"` //
SecretFile string `json:"secret_file"` // the file the server reads the secret key from
SecretFrom string `json:"secret_from,omitempty"` // a source to install it from, 0400; empty means already there
}
// Clustered says whether this machine carries cluster configuration.
func (m Machine) Clustered() bool { return m.Cluster != nil }
const Version = 2
var kinds = map[string]bool{"server": true, "console": true, "webmail": true}
var shapes = map[string]bool{"container": true, "host": true}
@@ -82,8 +141,8 @@ func Load(path string) (*Topology, error) {
// spread over machines, and the interesting refusals are about arrangements
// that cannot work rather than about syntax.
func (t *Topology) Validate() error {
if t.Version != Version {
return fmt.Errorf("version %d: this installer writes and reads version %d", t.Version, Version)
if t.Version < 1 || t.Version > Version {
return fmt.Errorf("version %d: this installer reads versions 1 to %d", t.Version, Version)
}
if t.Domain == "" {
return fmt.Errorf("no domain")
@@ -94,7 +153,7 @@ func (t *Topology) Validate() error {
return fmt.Errorf("no machines")
}
seen := map[string]bool{}
servers := 0
servers, clustered := 0, 0
for _, m := range t.Machines {
if m.Name == "" {
return fmt.Errorf("a machine has no name")
@@ -125,14 +184,70 @@ func (t *Topology) Validate() error {
servers++
}
}
if m.Cluster != nil {
if t.Version < 2 {
return fmt.Errorf("machine %q has a cluster block, which needs version 2", m.Name)
}
if !kindsHere["server"] {
return fmt.Errorf("machine %q has a cluster block but runs no mail server; only a server joins a cluster", m.Name)
}
if err := m.Cluster.validate(m.Name); err != nil {
return err
}
clustered++
}
}
switch servers {
case 1:
case 0:
switch {
case servers == 0:
return fmt.Errorf("no machine runs the mail server")
case servers == 1:
// One server, clustered or not, is always allowed.
case clustered == servers:
// Every mail node carries cluster configuration: a clustered install.
default:
return fmt.Errorf("%d machines run the mail server: one mail server for now, because a second "+
"node needs a shared store and cluster configuration, which this installer does not set up", servers)
return fmt.Errorf("%d machines run the mail server but %d carry a cluster block: "+
"a clustered install needs one on every mail node, or run a single server", servers, clustered)
}
return nil
}
// validate checks a cluster node describes a store the installer can point a
// server at and secrets it can place. The addresses themselves are the
// operator's to get right; what it refuses is a block that cannot be acted on.
func (c *ClusterNode) validate(machine string) error {
fail := func(f string, a ...any) error {
return fmt.Errorf("machine %q cluster: %s", machine, fmt.Sprintf(f, a...))
}
s := c.Store
switch {
case s.Type == "":
return fail("the store needs a type, e.g. \"PostgreSql\"")
case s.Host == "":
return fail("the store needs a host")
case s.Port <= 0 || s.Port > 65535:
return fail("the store port %d is not a port", s.Port)
case s.Database == "":
return fail("the store needs a database name")
case s.User == "":
return fail("the store needs a user")
case s.SecretFile == "":
return fail("the store needs a secret_file: where the server reads the password")
}
if c.Coordinator != nil {
if c.Coordinator.Addr == "" {
return fail("the coordinator needs an addr")
}
if c.Coordinator.SecretFile == "" {
return fail("the coordinator needs a secret_file")
}
}
if c.BlobStore != nil {
if c.BlobStore.Endpoint == "" {
return fail("the blob store needs an endpoint")
}
if c.BlobStore.SecretFile == "" {
return fail("the blob store needs a secret_file")
}
}
return nil
}