From bca479d29a08f90597178370db7b687fbdf6c453 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 16 Nov 2026 08:41:53 -0700 Subject: [PATCH 1/2] Give the server log files, so the console's Logs page has something (#70) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bootstrap takes one tracer and was given stdout, and the server adds its own file tracer only when it has none, so no installed server ever wrote a log file and Observability › Logs, which reads them, was always empty. A Log tracer is now added beside the stdout one with the setup's API key: the server's own default (inbuxa.log, rotated daily, plain text), deleted after inbuxa:LogSettings.keepForDays, which a new install sets to 30. On the host the files go to /var/log/inbuxa, which the unit makes with LogsDirectory= and the installer makes too; in a container, where that path is not a volume and the image's user could not write to one mounted there, to /var/lib/inbuxa/logs on the data volume. An install from before this is given its tracer by `inbuxa upgrade --key` or a run of install with --key, and is told so until then. A tracer an administrator set up is left alone, and a server moved between a container and the host takes the installer's tracer with it. A move to the host leaves the container's log files in its volume, and purge removes /var/log/inbuxa. --- README.md | 7 ++ cmd/inbuxa/completions.go | 2 +- cmd/inbuxa/main.go | 48 ++++++++++-- cmd/inbuxa/main_test.go | 1 + internal/apply/apply.go | 29 +++++++ internal/apply/hostserver.go | 30 +++++--- internal/apply/logfiles.go | 124 ++++++++++++++++++++++++++++++ internal/apply/logfiles_test.go | 60 +++++++++++++++ internal/apply/move.go | 5 +- internal/apply/uninstall.go | 4 + internal/jmap/client_test.go | 94 +++++++++++++++++++++++ internal/jmap/setup.go | 130 +++++++++++++++++++++++++++++++- internal/plan/plan.go | 3 +- internal/state/state.go | 5 ++ 14 files changed, 519 insertions(+), 23 deletions(-) create mode 100644 internal/apply/logfiles.go create mode 100644 internal/apply/logfiles_test.go diff --git a/README.md b/README.md index e97a988..269e883 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,13 @@ This is early. What is built: the server, which may convert its store on first start, it is stopped and its own data archived into `/backups` (`--no-backup` to skip; a store kept elsewhere, PostgreSQL or S3, is backed up where it lives). + A server installed before the installer turned on its log files (#70) + writes none, so the console's Logs page is empty: `--key -` with an API + key made in the console gives it them, upgrade due or not. They go to + `/var/log/inbuxa` on the host and `/var/lib/inbuxa/logs` on the data + volume in a container, one file a day, deleted after 30 days on a new + install (Settings › Security › Security, Log files). They hold IP and email + addresses. - **`uninstall [--purge]`** -- takes away what the installer put on this machine: containers, units, binaries (and the old ones kept for rollback), diff --git a/cmd/inbuxa/completions.go b/cmd/inbuxa/completions.go index 97adadc..7a7c727 100644 --- a/cmd/inbuxa/completions.go +++ b/cmd/inbuxa/completions.go @@ -47,7 +47,7 @@ func commands() []command { {name: "status", desc: "Report what is installed here, and whether it agrees"}, {name: "doctor", desc: "Run a read-only health check of this install", flags: []string{"timeout"}}, {name: "backup", desc: "Archive this machine's mail server to /backups", flags: []string{"yes"}}, - {name: "upgrade", desc: "Move what is here to the releases this installer installs", arg: "component", flags: []string{"dry-run", "yes", "no-backup", "check"}}, + {name: "upgrade", desc: "Move what is here to the releases this installer installs", arg: "component", flags: []string{"dry-run", "yes", "no-backup", "check", "key"}}, {name: "uninstall", desc: "Remove the programs, keep the data", flags: []string{"dry-run", "yes", "purge"}}, {name: "certs", desc: "Inspect or renew the mail server's certificate", flags: []string{"key", "mail-host", "email", "acme-directory", "dry-run"}}, {name: "dns", desc: "Print the DNS records the domain needs", flags: []string{"refresh", "key", "domain"}}, diff --git a/cmd/inbuxa/main.go b/cmd/inbuxa/main.go index c22aa91..6188d03 100644 --- a/cmd/inbuxa/main.go +++ b/cmd/inbuxa/main.go @@ -62,7 +62,10 @@ const usage = `inbuxa -- install the inbuxa suite on this machine inbuxa upgrade [component…] move what is here to the releases this installer installs: --dry-run, --yes, --no-backup, --check (report only; exit 10 if - anything can be upgraded) + anything can be upgraded), --key (an API key, + to give a server installed before log files + were turned on the ones the console's Logs + page reads; "-" reads it from standard input) inbuxa uninstall take the programs away, keep the data: --dry-run, --yes, --purge (the data too) inbuxa certs [--key KEY] the mail server's certificate: what its mail @@ -106,8 +109,10 @@ install flags: move the server between a container and the host (run install again with the other --server), and to move a --proxy snippets install made before - nginx passed the server's names through to it; - "-" reads it from standard input, and + nginx passed the server's names through to it, + or to give a server installed before log files + were turned on the ones the console's Logs page + reads; "-" reads it from standard input, and INBUXA_API_KEY works too --answers FILE read these options from a JSON file, for an unattended install (cloud-init, Ansible); keys @@ -892,6 +897,7 @@ func upgradeCmd(args []string) int { yes := fs.Bool("yes", false, "") noBackup := fs.Bool("no-backup", false, "") check := fs.Bool("check", false, "") + key := fs.String("key", os.Getenv("INBUXA_API_KEY"), "") // Components and options in any order: `upgrade server --yes` as well as // `upgrade --yes server`. The flag package stops at the first argument // that isn't an option, so parse again after each one. @@ -940,8 +946,14 @@ func upgradeCmd(args []string) int { } fmt.Printf(" %-8s %-10s %s\n", s.Component, s.Shape, mark) } - if due == 0 { + // A server installed before #70 writes no log files, and is given them + // here when there is an API key to do it with. + logFiles := apply.LogFilesDue(st) + if due == 0 && (*check || !logFiles || *key == "") { fmt.Println("\nNothing to do: this is what this installer installs. A newer installer brings newer releases.") + if logFiles { + fmt.Println("\nBut " + apply.LogFilesHint + ".") + } return 0 } // --check reports and stops, with an exit code a script or a cron job can @@ -951,6 +963,12 @@ func upgradeCmd(args []string) int { fmt.Printf("\n%d component(s) can be upgraded. Run `inbuxa upgrade` to apply.\n", due) return 10 } + switch { + case logFiles && *key == "": + fmt.Println("\nAfterwards, " + apply.LogFilesHint + ".") + case logFiles: + fmt.Println("\nThe server is also given the log files the console's Logs page reads, which it has not had.") + } backup := !*noBackup for _, s := range steps { if s.Component == plan.Server && s.Due() { @@ -973,10 +991,26 @@ func upgradeCmd(args []string) int { fmt.Fprintln(os.Stderr, "upgrading needs root") return 1 } + k := "" + if logFiles && *key != "" { + var err error + if k, err = readKey(*key); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + return 2 + } + } fmt.Println("\nUpgrading:") - if err := apply.RunUpgrade(context.Background(), st, steps, backup, f, &printer{}); err != nil { - fmt.Fprintln(os.Stderr, "\nstopped: "+err.Error()) - return 1 + if due > 0 { + if err := apply.RunUpgrade(context.Background(), st, steps, backup, f, &printer{}); err != nil { + fmt.Fprintln(os.Stderr, "\nstopped: "+err.Error()) + return 1 + } + } + if k != "" { + if err := apply.LogFiles(context.Background(), st, k, f, &printer{}); err != nil { + fmt.Fprintln(os.Stderr, "\nthe log files were not turned on: "+err.Error()) + return 1 + } } fmt.Println("\nDone.") return 0 diff --git a/cmd/inbuxa/main_test.go b/cmd/inbuxa/main_test.go index f3c0061..77ac5b9 100644 --- a/cmd/inbuxa/main_test.go +++ b/cmd/inbuxa/main_test.go @@ -22,6 +22,7 @@ func TestUpgradeTakesOptionsAfterComponents(t *testing.T) { {"webmail", "--dry-run", "--no-backup"}, {"--check"}, {"server", "--check"}, + {"--key", "k", "--dry-run"}, } { if code := upgradeCmd(args); code == 2 { t.Errorf("upgrade %v: taken as a usage error", args) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 41af27c..10bf453 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -335,6 +335,25 @@ func Run(ctx context.Context, p plan.Plan, f host.Facts, log Log) (*Result, erro log.Info("your web server still terminates TLS for the server's names: run this again with --key to pass them through to it instead (#48)") } res, err := converge(ctx, o, stack, cmp, hs, serverChanged, existing, f, log) + // An install from before #70 has no log files; a run with a key + // gives it them, and after a move points them at this shape's + // directory. + if err == nil && (o.Key != "" || LogFilesDue(existing)) { + if o.Key == "" { + log.Info("%s", LogFilesHint) + } else { + log.Step("turning on the server's log files") + srv := &jmap.Client{BaseURL: "http://" + hostFinalBind, Token: o.Key} + if lerr := turnOnLogFiles(ctx, srv, onHost, mv != nil, cmp, log); lerr != nil { + log.Info("could not turn on the log files the console's Logs page reads: %v", lerr) + } else if !existing.LogFiles { + existing.LogFiles = true + if serr := existing.Save(); serr != nil { + log.Info("could not write %s: %v", state.Path, serr) + } + } + } + } if err == nil && hw != nil { // Its files may be new, or what it reads at start; or it may // simply not be running. @@ -466,6 +485,15 @@ func Run(ctx context.Context, p plan.Plan, f host.Facts, log Log) (*Result, erro } }() + // Not a reason to stop: the mail works without them, and a later run + // with --key tries again. + log.Step("turning on the server's log files") + logFiles := true + if err := turnOnLogFiles(ctx, srv, onHost, false, cmp, log); err != nil { + logFiles = false + log.Info("could not turn on the log files the console's Logs page reads: %v", err) + } + if onHost { log.Step("fitting the server's listeners to the host") if err := srv.HostListeners(ctx, hostFinalBind, o.Local); err != nil { @@ -642,6 +670,7 @@ func Run(ctx context.Context, p plan.Plan, f host.Facts, log Log) (*Result, erro st.Machine, st.Dir, st.Domain, st.MailHost, st.Local = o.Machine, dir, o.Domain, o.MailHost, o.Local st.Runtime, st.Topology = f.Runtime.Kind, o.TopologyPath st.PassThrough = stack.PassThrough + st.LogFiles = logFiles st.Shapes = map[string]string{} for _, c := range []plan.Component{plan.Server, plan.Console, plan.Webmail} { if sh := o.Shape(c); sh != plan.Skip { diff --git a/internal/apply/hostserver.go b/internal/apply/hostserver.go index 8345b61..da5c35e 100644 --- a/internal/apply/hostserver.go +++ b/internal/apply/hostserver.go @@ -187,13 +187,27 @@ func (h hostServer) WriteEnv(vars map[string]string) (bool, error) { // WriteUnit writes the unit and says whether it changed. configured adds the // condition that a finished install keeps: no configuration file, no start. func (h hostServer) WriteUnit(ctx context.Context, configured bool) (bool, error) { + changed, err := writeIfChanged(hostUnitPath, h.unit(configured), 0o644) + if err != nil { + return false, err + } + if changed { + if err := runQuiet(ctx, "systemctl", "daemon-reload"); err != nil { + return false, err + } + } + return changed, runQuiet(ctx, "systemctl", "enable", "--quiet", HostServerUnit) +} + +// unit is the unit's text. +func (h hostServer) unit(configured bool) string { cond := "" if configured { cond = "# A server that has lost its configuration stays down rather than coming\n" + "# back in bootstrap mode.\n" + "ConditionPathExists=" + h.configPath() + "\n" } - unit := fmt.Sprintf(`# Written by the inbuxa installer. Changes here are replaced on its next run. + return fmt.Sprintf(`# Written by the inbuxa installer. Changes here are replaced on its next run. [Unit] Description=inbuxa mail server Documentation=https://docs.inbuxa.org/ @@ -211,6 +225,10 @@ WorkingDirectory=%s # bind them, and survives the binary being replaced, which setcap does not. AmbientCapabilities=CAP_NET_BIND_SERVICE CapabilityBoundingSet=CAP_NET_BIND_SERVICE +# The log files the console's Logs page reads: systemd makes +# /var/log/inbuxa at each start, owned by the user above. +LogsDirectory=inbuxa +LogsDirectoryMode=0750 NoNewPrivileges=true ProtectSystem=full ProtectHome=true @@ -223,16 +241,6 @@ RestartSec=5 [Install] WantedBy=multi-user.target `, cond, HostServerUser, HostServerUser, h.envPath(), HostServerBinary, h.configPath(), h.home()) - changed, err := writeIfChanged(hostUnitPath, unit, 0o644) - if err != nil { - return false, err - } - if changed { - if err := runQuiet(ctx, "systemctl", "daemon-reload"); err != nil { - return false, err - } - } - return changed, runQuiet(ctx, "systemctl", "enable", "--quiet", HostServerUnit) } // Start (re)starts the server for a step of first boot, or as it runs from diff --git a/internal/apply/logfiles.go b/internal/apply/logfiles.go new file mode 100644 index 0000000..a53d254 --- /dev/null +++ b/internal/apply/logfiles.go @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package apply + +// Log files, which the console's Logs page reads (#70). +// +// Bootstrap gives the server one tracer, stdout, and the server adds its own +// file tracer only when it has none, so an installed server used to write no +// files and the Logs page was always empty. A Log tracer is added beside the +// stdout one, with the server's own defaults: inbuxa.log, rotated daily, and +// deleted after inbuxa:LogSettings.keepForDays (30 on a new install). +// +// Where the files go depends on the shape: +// +// - on the host, /var/log/inbuxa, the server's own default. The unit's +// LogsDirectory= makes it at each start, owned by the server's user; it +// is made here too, so a server that is not restarted can write there +// at once. +// - in a container, /var/lib/inbuxa/logs on the data volume. The image +// has no /var/log/inbuxa and runs as an unprivileged user, so a volume +// mounted there would be root's and unwritable. The data volume is the +// server's own, and the server keeps its snapshots/ beside its store in +// the same way. +// +// Adding a tracer needs an API key. A new install has its setup key; an +// install from before this is given its tracer by a run with --key, install +// or apply again or `inbuxa upgrade`, and is told so until then. + +import ( + "context" + "fmt" + "os" + + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/docker" + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/host" + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/jmap" + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/plan" + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/state" +) + +const ( + hostLogDir = "/var/log/inbuxa" + containerLogDir = "/var/lib/inbuxa/logs" +) + +// LogFilesHint is said to an install that has no log files yet, when there +// is no API key to give it them with. +const LogFilesHint = "the console's Logs page stays empty until the server writes log files, which installs made before installer #70 do not: " + + "run `inbuxa upgrade --key -` once, with an API key made in the console, to turn them on" + +// logDirs is where the server's log files go in a shape, and where they go +// in the other one. +func logDirs(onHost bool) (dir, other string) { + if onHost { + return hostLogDir, containerLogDir + } + return containerLogDir, hostLogDir +} + +// turnOnLogFiles makes the log directory and has the server write its log +// files there. moved says the server has just moved between a container and +// the host, so the installer's tracer may still name the other shape's +// directory, which is then corrected. +func turnOnLogFiles(ctx context.Context, srv *jmap.Client, onHost, moved bool, cmp docker.Compose, log Log) error { + dir, other := logDirs(onHost) + if onHost { + if err := os.MkdirAll(dir, 0o750); err != nil { + return err + } + if err := runQuiet(ctx, "chown", HostServerUser+":"+HostServerUser, dir); err != nil { + return err + } + if err := os.Chmod(dir, 0o750); err != nil { + return err + } + } else if err := cmp.Run(ctx, "exec", "-T", "server", "mkdir", "-p", "-m", "0750", dir); err != nil { + return fmt.Errorf("making %s in the server's container: %w", dir, err) + } + var replaces []string + if moved { + replaces = []string{other} + } + change, err := srv.EnsureLogTracer(ctx, dir, replaces...) + if err != nil { + return err + } + switch change { + case jmap.LogTracerPresent: + log.Info("log files in %s already", dir) + case jmap.LogTracerAdded: + log.Info("log files in %s, one a day, for the console's Logs page", dir) + case jmap.LogTracerMoved: + log.Info("the server's log files are in %s now", dir) + case jmap.LogTracerKept: + log.Info("a log file tracer set up in the console is kept as it is") + } + return nil +} + +// LogFilesDue is whether the server installed here has not been given its +// log files yet. +func LogFilesDue(st *state.State) bool { + sh := plan.Shape(st.Shapes[string(plan.Server)]) + return (sh == plan.Host || sh == plan.Container) && !st.Removed() && !st.LogFiles +} + +// LogFiles gives an installed server its log files, with an API key, and +// records that it has them. +func LogFiles(ctx context.Context, st *state.State, key string, f host.Facts, log Log) error { + onHost := plan.Shape(st.Shapes[string(plan.Server)]) == plan.Host + docker.UseRuntime(f.Runtime) + cmp := docker.Compose{Dir: st.Dir, Runtime: f.Runtime} + log.Step("turning on the server's log files") + srv := &jmap.Client{BaseURL: "http://" + hostFinalBind, Token: key} + if err := turnOnLogFiles(ctx, srv, onHost, false, cmp, log); err != nil { + return err + } + st.LogFiles = true + if err := st.Save(); err != nil { + log.Info("could not write %s: %v", state.Path, err) + } + return nil +} diff --git a/internal/apply/logfiles_test.go b/internal/apply/logfiles_test.go new file mode 100644 index 0000000..f9af950 --- /dev/null +++ b/internal/apply/logfiles_test.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package apply + +import ( + "path/filepath" + "strings" + "testing" + "time" + + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/state" +) + +// Each shape has its own directory, and names the other's as the one a +// moved server's tracer is taken from. +func TestLogDirsFollowTheShape(t *testing.T) { + if dir, other := logDirs(true); dir != "/var/log/inbuxa" || other != containerLogDir { + t.Errorf("host: %s, %s", dir, other) + } + // On the data volume, which the image's user can write to, and in a + // directory of its own: x:Log reads every file in it. + if dir, other := logDirs(false); !strings.HasPrefix(dir, containerDataPath) || dir == strings.TrimSuffix(containerDataPath, "/") || other != hostLogDir { + t.Errorf("container: %s, %s", dir, other) + } + // What a move to the host leaves behind, as tar names it in the volume. + if got := "./data/" + strings.TrimPrefix(containerLogDir, containerDataPath); got != "./data/logs" { + t.Errorf("excluded %s", got) + } +} + +// The unit has systemd make the host's directory, under the same name. +func TestUnitMakesTheLogDirectory(t *testing.T) { + u := hostServer{deploy: "/var/lib/inbuxa"}.unit(true) + for _, want := range []string{"LogsDirectory=" + filepath.Base(hostLogDir) + "\n", "LogsDirectoryMode=0750\n", "User=" + HostServerUser + "\n"} { + if !strings.Contains(u, want) { + t.Errorf("unit lacks %q:\n%s", want, u) + } + } +} + +// Only a server installed here, still installed, and without its log files +// is due them. +func TestLogFilesDue(t *testing.T) { + now := time.Now() + for _, c := range []struct { + st state.State + due bool + }{ + {state.State{Shapes: map[string]string{"server": "container"}}, true}, + {state.State{Shapes: map[string]string{"server": "host", "webmail": "host"}}, true}, + {state.State{Shapes: map[string]string{"server": "host"}, LogFiles: true}, false}, + {state.State{Shapes: map[string]string{"console": "container", "webmail": "container"}, ServerURL: "https://mail.example.org"}, false}, + {state.State{Shapes: map[string]string{"server": "container"}, RemovedAt: &now}, false}, + } { + if got := LogFilesDue(&c.st); got != c.due { + t.Errorf("%+v: due = %v", c.st, got) + } + } +} diff --git a/internal/apply/move.go b/internal/apply/move.go index a4f5c62..b2d6b30 100644 --- a/internal/apply/move.go +++ b/internal/apply/move.go @@ -276,9 +276,12 @@ func (m *serverMove) toHost(ctx context.Context, stack compose.Stack, snippets b return back(err) } defer os.RemoveAll(stage) + // Not its log files (#70): on the host they go to /var/log/inbuxa, and + // a copy under the store would never be deleted. They stay in the + // volumes with the rest until those are removed. out := exec.CommandContext(ctx, docker.CLI, "run", "--rm", "--user", "0", "--entrypoint", "tar", "-v", "inbuxa_inbuxa-etc:/from/etc:ro", "-v", "inbuxa_inbuxa-data:/from/data:ro", - m.oldImage, "-cf", "-", "-C", "/from", ".") + m.oldImage, "-cf", "-", "-C", "/from", "--exclude=./data/"+strings.TrimPrefix(containerLogDir, containerDataPath), ".") in := exec.CommandContext(ctx, "tar", "-xf", "-", "-C", stage) if err := pipe(out, in); err != nil { return back(fmt.Errorf("copying the server's volumes: %w", err)) diff --git a/internal/apply/uninstall.go b/internal/apply/uninstall.go index d4a7ac1..e2656d3 100644 --- a/internal/apply/uninstall.go +++ b/internal/apply/uninstall.go @@ -67,6 +67,9 @@ func UninstallPlan(st *state.State, purge bool) Removal { if st.Shapes["server"] == string(plan.Host) { r.Remove = append(r.Remove, HostServerUnit+", "+HostServerBinary+" and any older binaries kept beside it") keepOrRemove(filepath.Join(dir, "server") + " (the server's configuration and mail) and its user, inbuxa") + if exists(hostLogDir) { + keepOrRemove(hostLogDir + " (the server's log files)") + } } if st.Shapes["console"] == string(plan.Host) { r.Remove = append(r.Remove, filepath.Join(dir, "console")+" (the console's files)") @@ -140,6 +143,7 @@ func Uninstall(ctx context.Context, st *state.State, purge bool, f host.Facts, l os.Remove(filepath.Join(dir, "server.env")) if purge { os.RemoveAll(filepath.Join(dir, "server")) + os.RemoveAll(hostLogDir) runQuiet(ctx, "userdel", HostServerUser) } } diff --git a/internal/jmap/client_test.go b/internal/jmap/client_test.go index f840331..6ea7a9f 100644 --- a/internal/jmap/client_test.go +++ b/internal/jmap/client_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "strings" @@ -267,3 +268,96 @@ func TestPassThroughListenerPutsBackAnHTTPSListener(t *testing.T) { t.Errorf("trusted = %v", nets) } } + +// tracers is an x:Tracer/query and /get answer listing ts. +func tracers(ts ...string) string { + ids := make([]string, len(ts)) + for i := range ts { + ids[i] = fmt.Sprintf("%q", fmt.Sprint(i)) + } + return `{"methodResponses":[["x:Tracer/query",{"ids":[` + strings.Join(ids, ",") + `]},"0"],` + + `["x:Tracer/get",{"list":[` + strings.Join(ts, ",") + `]},"1"]]}` +} + +const stdoutTracer = `{"id":"a","@type":"Stdout","enable":true,"level":"info"}` + +// An installed server has only the stdout tracer bootstrap gave it, so the +// Log tracer the Logs page reads is created beside it, as the server's own +// default would have been (#70). +func TestEnsureLogTracerAddsOneBesideStdout(t *testing.T) { + c, seen := fake(t, map[string]string{ + "x:Tracer/query": tracers(stdoutTracer), + "x:Tracer/set": `{"methodResponses":[["x:Tracer/set",{"created":{"log":{"id":"b"}}},"0"]]}`, + }) + got, err := c.EnsureLogTracer(context.Background(), "/var/lib/inbuxa/logs") + if err != nil || got != LogTracerAdded { + t.Fatalf("got %v, %v", got, err) + } + if len(*seen) != 2 { + t.Fatalf("%d requests", len(*seen)) + } + log := (*seen)[1]["args"].(map[string]any)["create"].(map[string]any)["log"].(map[string]any) + for k, want := range map[string]any{ + "@type": "Log", "enable": true, "path": "/var/lib/inbuxa/logs", + "prefix": "inbuxa.log", "rotate": "daily", "level": "info", "ansi": false, + } { + if log[k] != want { + t.Errorf("%s = %v, want %v", k, log[k], want) + } + } +} + +// Run again, nothing is written. +func TestEnsureLogTracerLeavesOneThatIsThere(t *testing.T) { + c, seen := fake(t, map[string]string{ + "x:Tracer/query": tracers(stdoutTracer, `{"id":"b","@type":"Log","enable":true,"path":"/var/log/inbuxa/","prefix":"inbuxa.log"}`), + }) + got, err := c.EnsureLogTracer(context.Background(), "/var/log/inbuxa", "/var/lib/inbuxa/logs") + if err != nil || got != LogTracerPresent || len(*seen) != 1 { + t.Fatalf("got %v, %v after %d requests", got, err, len(*seen)) + } +} + +// A server moved between a container and the host takes the installer's +// tracer with it, pointed at the directory its new shape has. +func TestEnsureLogTracerMovesTheInstallersOwn(t *testing.T) { + c, seen := fake(t, map[string]string{ + "x:Tracer/query": tracers(stdoutTracer, `{"id":"b","@type":"Log","enable":true,"path":"/var/lib/inbuxa/logs","prefix":"inbuxa.log"}`), + "x:Tracer/set": `{"methodResponses":[["x:Tracer/set",{"updated":{"b":null}},"0"]]}`, + }) + got, err := c.EnsureLogTracer(context.Background(), "/var/log/inbuxa", "/var/lib/inbuxa/logs") + if err != nil || got != LogTracerMoved { + t.Fatalf("got %v, %v", got, err) + } + update := (*seen)[1]["args"].(map[string]any)["update"].(map[string]any) + if b, ok := update["b"].(map[string]any); !ok || b["path"] != "/var/log/inbuxa" || len(update) != 1 { + t.Errorf("update = %v", update) + } +} + +// A Log tracer an administrator set up is theirs: one writing elsewhere, one +// turned off, or one in the other shape's directory under a prefix of their +// own is left alone, and nothing is added beside it. +func TestEnsureLogTracerKeepsAnAdministrators(t *testing.T) { + for _, log := range []string{ + `{"id":"b","@type":"Log","enable":true,"path":"/srv/logs","prefix":"inbuxa.log"}`, + `{"id":"b","@type":"Log","enable":false,"path":"/var/log/inbuxa","prefix":"inbuxa.log"}`, + `{"id":"b","@type":"Log","enable":true,"path":"/var/lib/inbuxa/logs","prefix":"mail"}`, + } { + c, seen := fake(t, map[string]string{"x:Tracer/query": tracers(stdoutTracer, log)}) + got, err := c.EnsureLogTracer(context.Background(), "/var/log/inbuxa", "/var/lib/inbuxa/logs") + if err != nil || got != LogTracerKept || len(*seen) != 1 { + t.Errorf("%s: got %v, %v after %d requests", log, got, err, len(*seen)) + } + } +} + +func TestEnsureLogTracerRefusalIsAnError(t *testing.T) { + c, _ := fake(t, map[string]string{ + "x:Tracer/query": tracers(stdoutTracer), + "x:Tracer/set": `{"methodResponses":[["x:Tracer/set",{"notCreated":{"log":{"type":"invalidProperties","properties":["path"]}}},"0"]]}`, + }) + if _, err := c.EnsureLogTracer(context.Background(), "/var/log/inbuxa"); err == nil || !strings.Contains(err.Error(), "path") { + t.Fatalf("err = %v", err) + } +} diff --git a/internal/jmap/setup.go b/internal/jmap/setup.go index b6b3099..ba6fdc6 100644 --- a/internal/jmap/setup.go +++ b/internal/jmap/setup.go @@ -55,8 +55,11 @@ func (c *Client) Bootstrap(ctx context.Context, hostname, domain, dataPath strin // certificates, so turning it on would only look like it had worked. "requestTlsCertificate": false, "generateDkimKeys": true, - // The default logs to /var/log/stalwart, which does not exist in the - // image and is not a volume. A container logs to stdout. + // Bootstrap takes one tracer, and stdout is the one every later step + // reads when something fails: the journal on the host, the + // container's log otherwise. The log files the console's Logs page + // reads are added beside it once there is an API key + // (EnsureLogTracer, #70). "tracer": map[string]any{ "@type": "Stdout", "enable": true, "level": "info", "ansi": false, "multiline": false, "lossy": false, @@ -549,3 +552,126 @@ func (c *Client) PassThroughListener(ctx context.Context, bind string, trusted [ _, err = DecodeSet(rs[0]) return err } + +// LogFilePrefix is what the installer's Log tracer names its files, and the +// server's own default: inbuxa.log.2026-11-16 and so on, one a day. +const LogFilePrefix = "inbuxa.log" + +// LogTracerChange is what EnsureLogTracer did. +type LogTracerChange int + +const ( + // LogTracerPresent: a Log tracer already writes to the path. + LogTracerPresent LogTracerChange = iota + // LogTracerAdded: there was none, and one was created. + LogTracerAdded + // LogTracerMoved: the installer's own tracer wrote to one of the paths + // it replaces, and now writes to the path. + LogTracerMoved + // LogTracerKept: a Log tracer someone else set up writes elsewhere, or + // is turned off, and was left as it is. + LogTracerKept +) + +// Tracer is as much of a tracer as EnsureLogTracer reads. +type Tracer struct { + ID string `json:"id"` + Type string `json:"@type"` + Path string `json:"path"` + Prefix string `json:"prefix"` + Enable bool `json:"enable"` +} + +// Tracers lists the server's tracers. +func (c *Client) Tracers(ctx context.Context) ([]Tracer, error) { + rs, err := c.Do(ctx, + Call{"x:Tracer/query", map[string]any{}, "0"}, + Call{"x:Tracer/get", map[string]any{ + "#ids": map[string]any{"resultOf": "0", "name": "x:Tracer/query", "path": "/ids"}, + }, "1"}, + ) + if err != nil { + return nil, err + } + var got struct { + List []Tracer `json:"list"` + } + return got.List, json.Unmarshal(rs[1].Args, &got) +} + +// EnsureLogTracer makes sure a Log tracer writes the server's log to files +// in dir, which is what the console's Logs page (x:Log) reads. The server +// adds one itself only when it has no tracer at all, and bootstrap gives it +// a stdout tracer, so without this an installed server never writes a file. +// +// The files rotate daily under LogFilePrefix; the server deletes them after +// inbuxa:LogSettings.keepForDays, which a new install sets to 30. +// +// A tracer an administrator set up is never changed: if any Log tracer is +// there and is not the installer's, it is left as it is. The installer's is +// told apart by its prefix and by writing to one of replaces, the path a +// server in the other shape uses; that one is moved to dir, which is what a +// server moved between a container and the host needs. +func (c *Client) EnsureLogTracer(ctx context.Context, dir string, replaces ...string) (LogTracerChange, error) { + ts, err := c.Tracers(ctx) + if err != nil { + return 0, err + } + same := func(a, b string) bool { return strings.TrimRight(a, "/") == strings.TrimRight(b, "/") } + var ours, other *Tracer + for i, t := range ts { + if t.Type != "Log" { + continue + } + if same(t.Path, dir) && t.Enable { + return LogTracerPresent, nil + } + mine := false + for _, r := range replaces { + mine = mine || (same(t.Path, r) && t.Prefix == LogFilePrefix) + } + if mine && ours == nil { + ours = &ts[i] + } else if !mine { + other = &ts[i] + } + } + var call Call + change := LogTracerAdded + switch { + case other != nil: + return LogTracerKept, nil + case ours != nil: + change = LogTracerMoved + call = Call{"x:Tracer/set", map[string]any{"update": map[string]any{ours.ID: map[string]any{"path": dir}}}, "0"} + default: + call = Call{"x:Tracer/set", map[string]any{"create": map[string]any{"log": logTracer(dir)}}, "0"} + } + rs, err := c.Do(ctx, call) + if err != nil { + return 0, err + } + s, err := DecodeSet(rs[0]) + if err != nil { + return 0, err + } + if change == LogTracerAdded { + if _, err := CreatedID(s, "log"); err != nil { + return 0, fmt.Errorf("x:Tracer/set: %w", err) + } + } + return change, nil +} + +// logTracer is the Log tracer the installer creates: the server's own +// default (crates/common/src/manager/defaults.rs), at dir. +func logTracer(dir string) map[string]any { + return map[string]any{ + "@type": "Log", "enable": true, "level": "info", + "path": dir, "prefix": LogFilePrefix, "rotate": "daily", + // Plain text: x:Log strips colour codes, but anyone reading the + // files with less or grep would not. + "ansi": false, "multiline": false, "lossy": false, + "events": map[string]any{}, "eventsPolicy": "exclude", + } +} diff --git a/internal/plan/plan.go b/internal/plan/plan.go index 20d2af3..be7099f 100644 --- a/internal/plan/plan.go +++ b/internal/plan/plan.go @@ -85,7 +85,8 @@ type Options struct { // Key is an API key for what changes a running server's own settings: // moving it between a container and the host, and fitting its HTTPS // listener to --proxy snippets' name router on an install from before - // it (#48). Never written down. + // it (#48), and giving a server installed before #70 its log files. + // Never written down. Key string // Where this came from, when it came from a topology file: the machine's diff --git a/internal/state/state.go b/internal/state/state.go index 9cbba66..ff3e55a 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -43,6 +43,11 @@ type State struct { // before it are moved over by a run with --key. PassThrough bool `json:"passThrough,omitempty"` + // LogFiles is whether the server was given the Log tracer the console's + // Logs page reads (#70). Installs from before it are given it by a run + // with --key. + LogFiles bool `json:"logFiles,omitempty"` + // Clustered is whether `inbuxa cluster join` pointed this node's server at // a shared store; ClusterRole is the role it was given, empty for the node // that carries every task. -- 2.54.0 From fe7e08e189d8985ef3f5433b19157337bbf1a915 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 16 Nov 2026 11:32:17 -0700 Subject: [PATCH 2/2] e2e: log files on a container install, a host install, and an upgrade with --key (#70) --- e2e/cases/install-host.sh | 9 ++++ e2e/cases/install-local.sh | 8 +++ e2e/cases/upgrade-log-files.sh | 91 ++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100755 e2e/cases/upgrade-log-files.sh diff --git a/e2e/cases/install-host.sh b/e2e/cases/install-host.sh index f161195..e7074ca 100644 --- a/e2e/cases/install-host.sh +++ b/e2e/cases/install-host.sh @@ -117,6 +117,15 @@ grep -q "server.*installed as a host" <<<"$STATUS" && ok "status: the server is [ $rc -eq 0 ] && ok "and intent and reality agree (exit 0)" || { bad "status exit $rc"; echo "$STATUS" | sed 's/^/ /'; } /tmp/inbuxa export 2>/dev/null | grep -A1 '"kind": "server"' | grep -q '"shape": "host"' && ok "export writes the server as host" || bad "export does not say host" +echo +echo "==> the server writes log files, for the console's Logs page (#70)" +logfile() { ls -t /var/log/inbuxa/inbuxa.log.* 2>/dev/null | head -1; } +for _ in $(seq 1 30); do [ -s "$(logfile)" ] && break; sleep 2; done +[ -s "$(logfile)" ] && ok "a log file in /var/log/inbuxa: $(basename "$(logfile)")" || bad "no inbuxa.log.* in /var/log/inbuxa" +[ "$(stat -c %U /var/log/inbuxa 2>/dev/null)" = inbuxa ] && ok "the directory belongs to the server's user" || bad "/var/log/inbuxa belongs to $(stat -c %U /var/log/inbuxa 2>/dev/null)" +systemctl show -p LogsDirectory $UNIT | grep -q "=inbuxa" && ok "and the unit keeps it (LogsDirectory)" || bad "the unit has no LogsDirectory" +grep -q '"logFiles": true' /etc/inbuxa/install.json && ok "the install record says it has them" || bad "the install record doesn't say logFiles" + echo echo "==> running it again converges rather than duplicating" OUT="$(/tmp/inbuxa install --local --domain example.test --server host --yes 2>&1)"; rc=$? diff --git a/e2e/cases/install-local.sh b/e2e/cases/install-local.sh index 9066f47..20a23ec 100644 --- a/e2e/cases/install-local.sh +++ b/e2e/cases/install-local.sh @@ -107,6 +107,14 @@ CODE=$(curl -s -o /tmp/login.json -b /tmp/jar -w '%{http_code}' -H 'X-Requested- [ "$CODE" = 200 ] && ok "sign-in succeeds" || { bad "the session answered $CODE"; head -c 200 /tmp/login.json | sed 's/^/ /'; } grep -q "urn:ietf:params:jmap:mail" /tmp/login.json && ok "and the session carries the mail capability" || bad "no mail capability in the session" +echo +echo "==> the server writes log files, for the console's Logs page (#70)" +VOL=$($(rt) volume inspect -f '{{.Mountpoint}}' "$($(rt) volume ls -q | grep 'inbuxa-data$' | head -1)" 2>/dev/null) +logfile() { ls -t "$VOL"/logs/inbuxa.log.* 2>/dev/null | head -1; } +for _ in $(seq 1 30); do [ -s "$(logfile)" ] && break; sleep 2; done +[ -s "$(logfile)" ] && ok "a log file in the server's /var/lib/inbuxa/logs: $(basename "$(logfile)")" || bad "no inbuxa.log.* in the data volume ($VOL/logs)" +grep -q '"logFiles": true' /etc/inbuxa/install.json && ok "the install record says it has them" || bad "the install record doesn't say logFiles" + echo echo "==> running it again converges rather than duplicating" OUT="$(/tmp/inbuxa install --local --domain example.test --yes 2>&1)"; rc=$? diff --git a/e2e/cases/upgrade-log-files.sh b/e2e/cases/upgrade-log-files.sh new file mode 100755 index 0000000..c9ae459 --- /dev/null +++ b/e2e/cases/upgrade-log-files.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# An install made before #70 writes no log files, so the console's Logs page +# stays empty. inbuxa upgrade says how to turn them on, and does it given an +# API key, whether or not there is anything else to upgrade. +# +# Run from the host with: OLD_REF= e2e/vm/run.sh e2e/cases/upgrade-log-files.sh +set -uo pipefail + +pass=0; fail=0 +ok() { echo " ok $*"; pass=$((pass+1)); } +bad() { echo " FAIL $*"; fail=$((fail+1)); } + +DIR=/var/lib/inbuxa +SERVER=http://127.0.0.1:8081 +[ -x /tmp/inbuxa-old ] || { echo "needs OLD_REF: an installer before #70"; exit 1; } + +rt() { command -v docker >/dev/null 2>&1 && echo docker || echo podman; } +compose() { + if [ "$(rt)" = docker ]; then + docker compose -f "$DIR/compose.yaml" "$@" + else + DOCKER_HOST=unix:///run/podman/podman.sock \ + /usr/local/lib/docker/cli-plugins/docker-compose -f "$DIR/compose.yaml" "$@" + fi +} +jmap() { # jmap TOKEN-or-user:pass JSON-CALLS + local auth + case "$1" in *:*) auth=(-u "$1") ;; *) auth=(-H "Authorization: Bearer $1") ;; esac + curl -s "${auth[@]}" -H 'Content-Type: application/json' \ + -d "{\"using\":[\"urn:ietf:params:jmap:core\",\"urn:inbuxa:jmap:registry\"],\"methodCalls\":$2}" "$SERVER/jmap/" +} +wait_server() { for _ in $(seq 1 60); do [ "$(curl -s -o /dev/null -w '%{http_code}' $SERVER/.well-known/jmap)" != 000 ] && return 0; sleep 2; done; return 1; } +logfile() { + local vol + vol=$($(rt) volume inspect -f '{{.Mountpoint}}' "$($(rt) volume ls -q | grep 'inbuxa-data$' | head -1)" 2>/dev/null) + ls -t "$vol"/logs/inbuxa.log.* 2>/dev/null | head -1 +} + +echo "==> installed by an installer from before #70" +OUT="$(/tmp/inbuxa-old install --local --domain example.test --install-deps --yes 2>&1)"; rc=$? +[ $rc -eq 0 ] && ok "installed with $(/tmp/inbuxa-old version)" || { bad "old install exit $rc"; echo "$OUT" | tail -8 | sed 's/^/ /'; } +sleep 10 +[ -z "$(logfile)" ] && ok "no log files" || bad "log files already: $(logfile)" + +echo +echo "==> upgrade without a key says how to turn them on" +OUT="$(/tmp/inbuxa upgrade --yes 2>&1)"; rc=$? +[ $rc -eq 0 ] && ok "upgrade exit 0" || { bad "upgrade exit $rc"; echo "$OUT" | tail -8 | sed 's/^/ /'; } +grep -q "inbuxa upgrade --key -" <<<"$OUT" && ok "it says to run it with --key" || { bad "no word of --key"; echo "$OUT" | tail -8 | sed 's/^/ /'; } +[ -z "$(logfile)" ] && ok "and turns nothing on" || bad "log files without a key: $(logfile)" + +echo +echo "==> an API key, as an administrator makes one in the console" +AUSER=$(awk '/^administrator/ {print $2}' "$DIR/credentials.txt") +APASS=$(awk '/^administrator/{getline; print $2}' "$DIR/credentials.txt") +printf 'services:\n server:\n environment:\n INBUXA_HTTP_BASIC_AUTH: all\n' > /tmp/basic.yaml +compose -f /tmp/basic.yaml up -d server >/dev/null 2>&1; wait_server +EXPIRES=$(date -u -d '+1 hour' +%Y-%m-%dT%H:%M:%SZ) +ANSWER=$(jmap "$AUSER:$APASS" "[[\"x:ApiKey/set\",{\"create\":{\"k\":{\"description\":\"log files test\",\"expiresAt\":\"$EXPIRES\",\"permissions\":{\"@type\":\"Inherit\"}}}},\"0\"]]") +KEY=$(sed -n 's/.*"secret":"\([^"]*\)".*/\1/p' <<<"$ANSWER") +compose up -d server >/dev/null 2>&1; wait_server +[ -n "$KEY" ] && ok "an API key for the administrator" || { bad "no API key"; head -c 300 <<<"$ANSWER" | sed 's/^/ /'; } + +echo +echo "==> upgrade with the key turns them on" +OUT="$(echo "$KEY" | /tmp/inbuxa upgrade --key - --yes 2>&1)"; rc=$? +echo "$OUT" | tail -6 | sed 's/^/ /' +[ $rc -eq 0 ] && ok "upgrade exit 0" || bad "upgrade exit $rc" +grep -q "log files in /var/lib/inbuxa/logs" <<<"$OUT" && ok "it says where they are" || bad "no word of where" +for _ in $(seq 1 30); do [ -s "$(logfile)" ] && break; sleep 2; done +[ -s "$(logfile)" ] && ok "a log file: $(basename "$(logfile)")" || bad "still no log file" +grep -q '"logFiles": true' /etc/inbuxa/install.json && ok "the install record says it has them" || bad "the install record doesn't say logFiles" +TRACERS=$(jmap "$KEY" '[["x:Tracer/query",{},"0"]]' | grep -o '"ids":\[[^]]*\]' | tr -cd ',' | wc -c) +ok "the server has $((TRACERS + 1)) tracer(s)" + +echo +echo "==> and once is enough" +OUT="$(/tmp/inbuxa upgrade --yes 2>&1)"; rc=$? +[ $rc -eq 0 ] && grep -q "Nothing to do" <<<"$OUT" && ok "nothing to do" || { bad "exit $rc"; echo "$OUT" | tail -6 | sed 's/^/ /'; } +grep -q -- "--key -" <<<"$OUT" && bad "it still asks for a key" || ok "and no more word of a key" +OUT="$(echo "$KEY" | /tmp/inbuxa install --local --domain example.test --key - --yes 2>&1)"; rc=$? +[ $rc -eq 0 ] && ok "installing again converges" || { bad "install again exit $rc"; echo "$OUT" | tail -6 | sed 's/^/ /'; } +N=$(jmap "$KEY" '[["x:Tracer/query",{},"0"]]' | grep -o '"ids":\[[^]]*\]' | tr -cd ',' | wc -c) +[ "$N" = "$TRACERS" ] && ok "without adding another tracer" || bad "tracers went from $((TRACERS + 1)) to $((N + 1))" + +echo +echo "==> $pass passed, $fail failed" +[ $fail -eq 0 ] -- 2.54.0