ci / test (pull_request) Successful in 1m28s
The check against links in the server's directory also refused a secret_file with no secret_from, which the join only checks is there and never reads or writes: only the server reads it. An operator may keep such a secret elsewhere and link it in, so it is no longer refused; a secret the join installs still is. Test: TestJoinLeavesALinkedSecretItDoesNotWrite is new and passes; TestJoinRefusesLinksInTheServersDirectory still passes.
370 lines
12 KiB
Go
370 lines
12 KiB
Go
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"os/user"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
|
|
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/topology"
|
|
)
|
|
|
|
// recorder is a Run stub that remembers the commands Apply ran, so a test can
|
|
// check that it reloaded and restarted without touching the real systemd.
|
|
type recorder struct{ cmds []string }
|
|
|
|
func (r *recorder) run(_ context.Context, name string, args ...string) error {
|
|
r.cmds = append(r.cmds, name+" "+strings.Join(args, " "))
|
|
return nil
|
|
}
|
|
|
|
// fixture lays out a server directory with a password source and an existing
|
|
// config.json, and returns a Config rooted entirely in a temp tree.
|
|
func fixture(t *testing.T, node topology.ClusterNode) (Config, *recorder) {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
etc := filepath.Join(root, "server")
|
|
if err := os.MkdirAll(etc, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// An existing single-node config, so a backup is made.
|
|
if err := os.WriteFile(filepath.Join(etc, "config.json"), []byte(`{"@type":"RocksDb"}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rec := &recorder{}
|
|
return Config{
|
|
Node: node,
|
|
Machine: "mx2",
|
|
EtcDir: etc,
|
|
Unit: "inbuxa-server.service",
|
|
UnitDir: filepath.Join(root, "systemd"),
|
|
User: "", // no chown in tests: ownership needs root
|
|
Run: rec.run,
|
|
}, rec
|
|
}
|
|
|
|
// clusterNode is a full cluster block with the password sources written to
|
|
// disk, for a happy-path test.
|
|
func clusterNode(t *testing.T, dir string) topology.ClusterNode {
|
|
t.Helper()
|
|
for name, body := range map[string]string{"pg.src": "pgpw", "nats.src": "natspw", "s3.src": "s3key"} {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return topology.ClusterNode{
|
|
Store: topology.ClusterStore{
|
|
Type: "PostgreSql", Host: "10.0.0.2", Port: 5000, Database: "inbuxa", User: "inbuxa",
|
|
SecretFile: "pg.pw", SecretFrom: filepath.Join(dir, "pg.src"),
|
|
},
|
|
Coordinator: &topology.ClusterService{
|
|
Addr: "10.0.0.2:4223", User: "inbuxa",
|
|
SecretFile: "nats.pw", SecretFrom: filepath.Join(dir, "nats.src"),
|
|
},
|
|
BlobStore: &topology.ClusterBlob{
|
|
Endpoint: "http://10.0.0.2:3910", Bucket: "blobs",
|
|
SecretFile: "s3.secret", SecretFrom: filepath.Join(dir, "s3.src"),
|
|
},
|
|
Role: "peer",
|
|
After: []string{"haproxy.service", "wg.service"},
|
|
}
|
|
}
|
|
|
|
func TestBuildPlanAndApply(t *testing.T) {
|
|
src := t.TempDir()
|
|
node := clusterNode(t, src)
|
|
cfg, rec := fixture(t, node)
|
|
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.Backup == "" {
|
|
t.Error("an existing config.json should be kept aside for rollback")
|
|
}
|
|
if err := Apply(context.Background(), p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// The store config points at the shared store, with the server's own field
|
|
// names and a File secret at the installed path.
|
|
b, err := os.ReadFile(cfg.configPath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got map[string]any
|
|
if err := json.Unmarshal(b, &got); err != nil {
|
|
t.Fatalf("config.json is not JSON: %v", err)
|
|
}
|
|
if got["@type"] != "PostgreSql" || got["host"] != "10.0.0.2" || got["poolMaxConnections"].(float64) != 40 {
|
|
t.Errorf("store config wrong: %v", got)
|
|
}
|
|
as, _ := got["authSecret"].(map[string]any)
|
|
if as["@type"] != "File" || as["filePath"] != filepath.Join(cfg.EtcDir, "pg.pw") {
|
|
t.Errorf("authSecret should be a File at the installed path, got %v", as)
|
|
}
|
|
|
|
// The three secrets are installed, each 0400.
|
|
for _, f := range []string{"pg.pw", "nats.pw", "s3.secret"} {
|
|
fi, err := os.Stat(filepath.Join(cfg.EtcDir, f))
|
|
if err != nil {
|
|
t.Fatalf("secret %s not installed: %v", f, err)
|
|
}
|
|
if fi.Mode().Perm() != 0o400 {
|
|
t.Errorf("%s mode is %v, want 0400", f, fi.Mode().Perm())
|
|
}
|
|
}
|
|
|
|
// The rollback copy holds the old config.
|
|
if b, _ := os.ReadFile(p.Backup); !strings.Contains(string(b), "RocksDb") {
|
|
t.Error("the backup should hold the previous config")
|
|
}
|
|
|
|
// The drop-ins carry the hostname, the role and the boot order.
|
|
dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf"))
|
|
if !strings.Contains(string(dropIn), "INBUXA_HOSTNAME=mx2") || !strings.Contains(string(dropIn), "INBUXA_ROLE=peer") {
|
|
t.Errorf("cluster.conf wrong:\n%s", dropIn)
|
|
}
|
|
order, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster-order.conf"))
|
|
if !strings.Contains(string(order), "After=haproxy.service wg.service") {
|
|
t.Errorf("cluster-order.conf wrong:\n%s", order)
|
|
}
|
|
|
|
// It reloaded systemd and restarted the server, and nothing else.
|
|
want := []string{"systemctl daemon-reload", "systemctl restart inbuxa-server.service"}
|
|
if strings.Join(rec.cmds, "|") != strings.Join(want, "|") {
|
|
t.Errorf("commands run: %v, want %v", rec.cmds, want)
|
|
}
|
|
}
|
|
|
|
func TestNoRoleOmitsRoleLine(t *testing.T) {
|
|
src := t.TempDir()
|
|
node := clusterNode(t, src)
|
|
node.Role = "" // the node that carries everything
|
|
cfg, _ := fixture(t, node)
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := Apply(context.Background(), p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dropIn, _ := os.ReadFile(filepath.Join(cfg.dropInDir(), "cluster.conf"))
|
|
if strings.Contains(string(dropIn), "INBUXA_ROLE") {
|
|
t.Errorf("a node with no role should get no INBUXA_ROLE line:\n%s", dropIn)
|
|
}
|
|
}
|
|
|
|
func TestDryRunWritesNothing(t *testing.T) {
|
|
src := t.TempDir()
|
|
cfg, rec := fixture(t, clusterNode(t, src))
|
|
cfg.DryRun = true
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := Apply(context.Background(), p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// config.json is untouched (still the RocksDb one), no secret was written,
|
|
// no drop-in directory, and no command ran.
|
|
if b, _ := os.ReadFile(cfg.configPath()); !strings.Contains(string(b), "RocksDb") {
|
|
t.Error("a dry run must not rewrite config.json")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(cfg.EtcDir, "pg.pw")); err == nil {
|
|
t.Error("a dry run must not install secrets")
|
|
}
|
|
if _, err := os.Stat(cfg.dropInDir()); err == nil {
|
|
t.Error("a dry run must not create the drop-in directory")
|
|
}
|
|
if len(rec.cmds) != 0 {
|
|
t.Errorf("a dry run must run no commands, ran %v", rec.cmds)
|
|
}
|
|
}
|
|
|
|
func TestMissingSecretSource(t *testing.T) {
|
|
cfg, _ := fixture(t, topology.ClusterNode{
|
|
Store: topology.ClusterStore{
|
|
Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa",
|
|
SecretFile: "pg.pw", // no SecretFrom and not on disk
|
|
},
|
|
})
|
|
if _, err := BuildPlan(cfg); err == nil || !strings.Contains(err.Error(), "secret_from") {
|
|
t.Errorf("a secret with no source and no file should fail, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSecretAlreadyInPlace(t *testing.T) {
|
|
cfg, _ := fixture(t, topology.ClusterNode{
|
|
Store: topology.ClusterStore{
|
|
Type: "PostgreSql", Host: "h", Port: 5000, Database: "inbuxa", User: "inbuxa",
|
|
SecretFile: "pg.pw", // already there, no source needed
|
|
},
|
|
})
|
|
if err := os.WriteFile(filepath.Join(cfg.EtcDir, "pg.pw"), []byte("x"), 0o400); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatalf("a secret already in place should be fine: %v", err)
|
|
}
|
|
for _, w := range p.Writes {
|
|
if w.Secret {
|
|
t.Error("a secret already in place should not be re-installed")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNoServerDir(t *testing.T) {
|
|
cfg := Config{Node: topology.ClusterNode{}, EtcDir: filepath.Join(t.TempDir(), "absent"), Unit: "u", UnitDir: "d"}
|
|
if _, err := BuildPlan(cfg); err == nil {
|
|
t.Error("a join before the server is installed should fail")
|
|
}
|
|
}
|
|
|
|
// victimFile is a file of root's outside the server's directory, which a
|
|
// link planted in that directory points at.
|
|
func victimFile(t *testing.T) string {
|
|
t.Helper()
|
|
p := filepath.Join(t.TempDir(), "shadow")
|
|
if err := os.WriteFile(p, []byte("root:secret-hash\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func untouched(t *testing.T, path string) {
|
|
t.Helper()
|
|
if b, _ := os.ReadFile(path); string(b) != "root:secret-hash\n" {
|
|
t.Errorf("%s was written through a link: %q", path, b)
|
|
}
|
|
}
|
|
|
|
// The server's directory is its user's, so that user can replace config.json
|
|
// or a secret with a symbolic link. A join runs as root and must neither
|
|
// write through such a link nor copy what it points at into the rollback
|
|
// copy (INSTALLER-1).
|
|
func TestJoinRefusesLinksInTheServersDirectory(t *testing.T) {
|
|
for _, name := range []string{"config.json", "config.json.prev", "pg.pw"} {
|
|
t.Run(name, func(t *testing.T) {
|
|
cfg, _ := fixture(t, clusterNode(t, t.TempDir()))
|
|
victim := victimFile(t)
|
|
// A link to a file not there yet, for the one a join only
|
|
// writes when it is missing.
|
|
absent := filepath.Join(filepath.Dir(victim), "cron.d-x")
|
|
link := filepath.Join(cfg.EtcDir, name)
|
|
os.Remove(link)
|
|
target := victim
|
|
if name == "config.json.prev" {
|
|
target = absent
|
|
}
|
|
if err := os.Symlink(target, link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p, err := BuildPlan(cfg)
|
|
if err == nil {
|
|
err = Apply(context.Background(), p)
|
|
}
|
|
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
|
|
t.Errorf("a link at %s should be refused, got %v", name, err)
|
|
}
|
|
untouched(t, victim)
|
|
if _, err := os.Stat(absent); err == nil {
|
|
t.Errorf("%s was created through a link", absent)
|
|
}
|
|
prev := cfg.configPath() + ".prev"
|
|
if fi, err := os.Lstat(prev); err == nil && fi.Mode().IsRegular() {
|
|
if b, _ := os.ReadFile(prev); strings.Contains(string(b), "secret-hash") {
|
|
t.Error("the rollback copy holds the link's target")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A link swapped in after the plan was made, while the join runs, is
|
|
// replaced, never written through.
|
|
func TestJoinDoesNotWriteThroughALinkPlantedAfterThePlan(t *testing.T) {
|
|
cfg, _ := fixture(t, clusterNode(t, t.TempDir()))
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
victim := victimFile(t)
|
|
for _, name := range []string{"pg.pw", "nats.pw"} {
|
|
if err := os.Symlink(victim, filepath.Join(cfg.EtcDir, name)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := Apply(context.Background(), p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
untouched(t, victim)
|
|
for _, name := range []string{"pg.pw", "nats.pw"} {
|
|
fi, err := os.Lstat(filepath.Join(cfg.EtcDir, name))
|
|
if err != nil || !fi.Mode().IsRegular() || fi.Mode().Perm() != 0o400 {
|
|
t.Errorf("%s should be a plain file, mode 0400: %v %v", name, fi, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// With the server's user named, what a join writes beside config.json is
|
|
// given to it. The current user stands in for it, so this runs without root.
|
|
func TestJoinGivesItsFilesToTheServersUser(t *testing.T) {
|
|
me, err := user.Current()
|
|
if err != nil {
|
|
t.Skip(err)
|
|
}
|
|
cfg, _ := fixture(t, clusterNode(t, t.TempDir()))
|
|
cfg.User = me.Username
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := Apply(context.Background(), p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"config.json", "config.json.prev", "pg.pw"} {
|
|
fi, err := os.Lstat(filepath.Join(cfg.EtcDir, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st := fi.Sys().(*syscall.Stat_t); strconv.Itoa(int(st.Uid)) != me.Uid {
|
|
t.Errorf("%s belongs to uid %d, want %s", name, st.Uid, me.Uid)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A secret already in place that the join leaves alone may be a link the
|
|
// operator made: only the server reads it, so it is not refused.
|
|
func TestJoinLeavesALinkedSecretItDoesNotWrite(t *testing.T) {
|
|
node := clusterNode(t, t.TempDir())
|
|
cfg, _ := fixture(t, node)
|
|
kept := filepath.Join(t.TempDir(), "pg.pw")
|
|
if err := os.WriteFile(kept, []byte("pgpw"), 0o400); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(kept, filepath.Join(cfg.EtcDir, "pg.pw")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg.Node.Store.SecretFrom = ""
|
|
p, err := BuildPlan(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := Apply(context.Background(), p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi, err := os.Lstat(filepath.Join(cfg.EtcDir, "pg.pw")); err != nil || fi.Mode()&os.ModeSymlink == 0 {
|
|
t.Errorf("the operator's link should be left as it is: %v %v", fi, err)
|
|
}
|
|
}
|