From 8725d8c11b10fc19883fac7be72d6ce4b7e8b81f Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 22 Sep 2026 19:11:52 -0700 Subject: [PATCH] Obtain certificates, and wait for the one that matters The public shape now works end to end: real ports, Caddy in front, and both programs that need certificates getting them from the same CA -- Caddy for the front ends over TLS-ALPN-01, the mail server for its own names over HTTP-01, which Caddy forwards on port 80. Proved in the lab against Pebble, with a DNS stub answering every name with the machine's own address, so no public name or public CA is involved: twenty checks, ending with IMAPS and submissions presenting a certificate for the mail host that verifies against the CA, and the webmail sending sign-in to the server as the first-party client the server registered. Two things the test found, both of which would have shipped: - The proxy fronted four of the server's five names. The server puts ua-auto-config in its own certificate too, so its challenge was never forwarded, one name failed, and the whole order failed with it -- leaving the mail ports on a self-signed certificate while everything else looked healthy. The list now matches what the server asks for. - Nothing waited for the certificate. An order that fails is not retried on its own and a restart does not start a new one, so the install declared itself finished over a self-signed certificate. It now waits, asks again every 45 seconds, and reports the issuer -- or says plainly that the server will keep trying once the domain resolves here, which is the ordinary case on a first install. --acme-directory and --acme-ca-root are what let a private CA be used: the root is added to the server image's own bundle and given to Caddy, because neither sees the other's trust store. --- README.md | 8 +- cmd/inbuxa/main.go | 8 ++ e2e/cases/install-public.sh | 153 ++++++++++++++++++++++++++++++++++++ internal/apply/apply.go | 91 ++++++++++++++++++--- internal/compose/compose.go | 11 ++- internal/plan/plan.go | 6 ++ 6 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 e2e/cases/install-public.sh diff --git a/README.md b/README.md index 5a7782d..e2f123d 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,10 @@ the installer draft); the phases are there too. inbuxa install --local --domain example.test --install-deps --yes is the shortest thing that works today: the whole suite on loopback, with no -DNS and no certificates, on a machine that starts with nothing. +DNS and no certificates, on a machine that starts with nothing. Without +`--local` it takes the real ports, puts Caddy in front and obtains +certificates -- which `e2e/cases/install-public.sh` proves against a private +CA, with no internet and no public name involved. ## Building and testing @@ -55,7 +58,8 @@ is tested on a throwaway virtual machine rather than on anybody's desk: e2e/vm/up.sh a Debian 13 machine, in qemu, as you e2e/vm/run.sh e2e/cases/survey.sh what it says about a machine e2e/vm/run.sh e2e/cases/deps.sh the offer, and taking it - e2e/vm/run.sh e2e/cases/install-local.sh a whole suite, and signing in to it + e2e/vm/run.sh e2e/cases/install-local.sh a whole suite, and signing in to it + e2e/vm/run.sh e2e/cases/install-public.sh the same with real ports and certificates e2e/vm/down.sh remove it Each case starts from a copy of the machine taken when it was new, so a run diff --git a/cmd/inbuxa/main.go b/cmd/inbuxa/main.go index 4337f91..2664ee2 100644 --- a/cmd/inbuxa/main.go +++ b/cmd/inbuxa/main.go @@ -46,6 +46,9 @@ install flags: --proxy WHICH caddy | snippets | none (default: caddy) --dir PATH where the installation lives; default: /var/lib/inbuxa --local loopback evaluation: no public ports, no certificates + --acme-directory URL a private ACME CA, for testing the certificate path + --acme-ca-root PATH that CA's root, which both the server and the proxy + are made to trust --install-deps install what the chosen shapes need and this machine lacks, rather than refusing over it --dry-run print the plan and stop @@ -107,6 +110,8 @@ func install(args []string) int { fs.StringVar(&o.Dir, "dir", "", "") fs.BoolVar(&o.Local, "local", false, "") fs.BoolVar(&o.InstallDeps, "install-deps", false, "") + fs.StringVar(&o.ACMEDirectory, "acme-directory", "", "") + fs.StringVar(&o.ACMECARoot, "acme-ca-root", "", "") if err := fs.Parse(args); err != nil { return 2 } @@ -164,6 +169,9 @@ func install(args []string) int { if res.WebmailURL != "" { fmt.Printf(" webmail %s\n", res.WebmailURL) } + if res.Certificate != "" { + fmt.Printf(" certificate issued by %s\n", res.Certificate) + } if res.ZoneFile != "" { fmt.Printf(" dns records %s\n", res.ZoneFile) } diff --git a/e2e/cases/install-public.sh b/e2e/cases/install-public.sh new file mode 100644 index 0000000..8d1f110 --- /dev/null +++ b/e2e/cases/install-public.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# The public shape, with no internet involved: Pebble stands in for Let's +# Encrypt and a DNS stub answers every name with this machine's address, so +# both Caddy and the mail server really obtain certificates, over the real +# ports, through the real Caddyfile. +# +# --local proves the pieces talk to each other. This proves the part an +# operator actually gets wrong: ports 25 and 443 held for real, two programs +# asking the same CA for certificates for overlapping names, and a proxy in +# front of all of it. The two are kept apart by challenge type -- Caddy uses +# TLS-ALPN-01 on 443, the server HTTP-01 on 80, which Caddy forwards -- and +# this is where that is checked. +# +# Run from the host with: e2e/vm/run.sh e2e/cases/install-public.sh +set -uo pipefail + +pass=0; fail=0 +ok() { echo " ok $*"; pass=$((pass+1)); } +bad() { echo " FAIL $*"; fail=$((fail+1)); } + +DOMAIN=lab.test +MAIL=mx.lab.test # not "mail": proves the names follow --mail-host +CONSOLE=console.lab.test +WEBMAIL=webmail.lab.test +DIR=/var/lib/inbuxa +WORK=/tmp/lab +LABNET=inbuxa-e2e +LABSUBNET=172.31.254.0/24 +HOSTIP=172.31.254.1 # this machine, as the lab network sees it + +rm -rf "$WORK"; mkdir -p "$WORK" + +echo "==> what the installer needs, before the lab" +/tmp/inbuxa deps --console container --webmail container --install >/dev/null 2>&1 || true +docker version >/dev/null 2>&1 && ok "docker is usable" || { bad "no docker"; exit 1; } + +echo +echo "==> standing up a private CA and a DNS stub" +docker network create --subnet "$LABSUBNET" "$LABNET" >/dev/null 2>&1 +# Pebble's own certificate names localhost and "pebble"; the server and Caddy +# reach it at this machine's address from another network, so it gets one for +# that address, signed by the test root that ships with it. +docker create --name inbuxa-e2e-extract ghcr.io/letsencrypt/pebble:latest >/dev/null 2>&1 +docker cp inbuxa-e2e-extract:/test/certs "$WORK/pebble-certs" >/dev/null +docker cp inbuxa-e2e-extract:/test/config/pebble-config.json "$WORK/pebble-config.json" >/dev/null +docker rm inbuxa-e2e-extract >/dev/null +openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -subj "/CN=pebble" \ + -keyout "$WORK/pebble-key.pem" -out "$WORK/pebble.csr" 2>/dev/null +openssl x509 -req -in "$WORK/pebble.csr" -days 2 -CA "$WORK/pebble-certs/pebble.minica.pem" \ + -CAkey "$WORK/pebble-certs/pebble.minica.key.pem" -CAcreateserial \ + -extfile <(printf 'subjectAltName=IP:%s\nextendedKeyUsage=serverAuth\n' "$HOSTIP") \ + -out "$WORK/pebble-cert.pem" 2>/dev/null +python3 - "$WORK/pebble-config.json" <<'PY' +import json, sys +c = json.load(open(sys.argv[1])) +c["pebble"].update(httpPort=80, tlsPort=443, + certificate="/work/pebble-cert.pem", privateKey="/work/pebble-key.pem") +json.dump(c, open(sys.argv[1], "w")) +PY +chmod -R a+r "$WORK" +docker run -d --name inbuxa-e2e-dns --network "$LABNET" --ip 172.31.254.3 \ + ghcr.io/letsencrypt/pebble-challtestsrv:latest \ + -defaultIPv4 "$HOSTIP" -defaultIPv6 "" -http01 "" -https01 "" -tlsalpn01 "" -doh "" >/dev/null +# Nonce rejection off: Pebble refuses 5% of nonces on purpose, and the server +# gives up an order on the first one rather than retrying. +docker run -d --name inbuxa-e2e-pebble --network "$LABNET" --ip 172.31.254.2 \ + -p 14000:14000 -p 15000:15000 -v "$WORK:/work:ro" \ + -e PEBBLE_VA_NOSLEEP=1 -e PEBBLE_WFE_NONCEREJECT=0 \ + ghcr.io/letsencrypt/pebble:latest -config /work/pebble-config.json -dnsserver 172.31.254.3:8053 >/dev/null +for _ in $(seq 1 30); do + curl -sf --cacert "$WORK/pebble-certs/pebble.minica.pem" "https://$HOSTIP:14000/dir" >/dev/null && break + sleep 1 +done +curl -sf --cacert "$WORK/pebble-certs/pebble.minica.pem" "https://$HOSTIP:14000/dir" >/dev/null \ + && ok "Pebble answers at https://$HOSTIP:14000/dir" || { bad "Pebble did not come up"; exit 1; } + +echo +echo "==> installing the public shape" +OUT="$(/tmp/inbuxa install --domain "$DOMAIN" --mail-host "$MAIL" --console-host "$CONSOLE" \ + --webmail-host "$WEBMAIL" --acme-directory "https://$HOSTIP:14000/dir" \ + --acme-ca-root "$WORK/pebble-certs/pebble.minica.pem" --yes 2>&1)"; rc=$? +echo "$OUT" | grep -v '^ |' | tail -22 | sed 's/^/ /' +[ $rc -eq 0 ] && ok "installed (exit 0)" || bad "exit $rc" + +echo +echo "==> the ports an operator would expect" +for p in 25 80 443 465 993 995 4190; do + ss -ltn "sport = :$p" | grep -q LISTEN && ok "$p is listening" || bad "$p is not listening" +done + +# Caddy obtains its certificates in the background, so give the first +# handshake for each name a little room. +expect() { + local want=$1 got=; shift + for _ in $(seq 1 40); do + got=$(curl -s -o /dev/null -w '%{http_code}' "$@" 2>/dev/null || true) + [ "$got" = "$want" ] && return 0 + sleep 1 + done + echo " got HTTP ${got:-nothing}, wanted $want" >&2 + return 1 +} + +CA="$WORK/pebble-certs/pebble.minica.pem" +curl -sf --cacert "$CA" "https://$HOSTIP:15000/roots/0" > "$WORK/root.pem" +curl -sf --cacert "$CA" "https://$HOSTIP:15000/intermediates/0" > "$WORK/int.pem" +cat "$WORK/int.pem" "$WORK/root.pem" > "$WORK/chain.pem" + +echo +echo "==> the front ends, over HTTPS, with certificates from the CA" +expect 200 --cacert "$WORK/chain.pem" --resolve "$WEBMAIL:443:127.0.0.1" "https://$WEBMAIL/api/health" \ + && ok "the webmail answers over HTTPS" || bad "the webmail does not answer over HTTPS" +expect 200 --cacert "$WORK/chain.pem" --resolve "$CONSOLE:443:127.0.0.1" "https://$CONSOLE/" \ + && ok "the console answers over HTTPS" || bad "the console does not answer over HTTPS" +expect 308 --cacert "$WORK/chain.pem" --resolve "$WEBMAIL:80:127.0.0.1" "http://$WEBMAIL/" \ + && ok "port 80 redirects" || bad "port 80 does not redirect" + +echo +echo "==> the mail server's own certificate, on the mail ports" +grep -q "certificate issued by" <<<"$OUT" && ok "the install reported a certificate" || bad "the install reported no certificate" +for port in 993 465; do + out=$(echo | timeout 20 openssl s_client -connect "127.0.0.1:$port" -servername "$MAIL" \ + -verify_hostname "$MAIL" -CAfile "$WORK/chain.pem" 2>&1) + grep -q "Verify return code: 0 (ok)" <<<"$out" \ + && ok "$port presents a certificate for $MAIL, issued by the CA" \ + || { bad "$port did not verify"; grep -E "Verify return code|subject=|issuer=" <<<"$out" | sed 's/^/ /'; } +done + +echo +echo "==> sign-in goes to the mail server's own page, as a registered client" +# Unlike --local, this shape has OAuth: the webmail refuses a password of its +# own and sends the browser to the server, as the first-party client the +# server registered for this URL. A 302 to the mail host is that working. +USER=$(awk '/^first mailbox/ {print $3}' "$DIR/credentials.txt") +LOC=$(curl -s -o /dev/null -w '%{redirect_url}' --cacert "$WORK/chain.pem" \ + --resolve "$WEBMAIL:443:127.0.0.1" --resolve "$MAIL:443:127.0.0.1" \ + "https://$WEBMAIL/api/auth/oauth/start?username=$USER&remember=1") +grep -q "^https://$MAIL/" <<<"$LOC" && ok "the webmail sends sign-in to $MAIL" || bad "sign-in went to '${LOC:-nowhere}'" +grep -q "client_id=ihasmail-inbuxa" <<<"$LOC" && ok "as the first-party client" || bad "no first-party client id in '$LOC'" +CODE=$(curl -s -o /dev/null -w '%{http_code}' --cacert "$WORK/chain.pem" --resolve "$MAIL:443:127.0.0.1" "$LOC") +[ "$CODE" = 200 ] && ok "and the server serves that page over its own certificate" || bad "the server answered $CODE for the sign-in page" + +echo +echo "==> and nothing was left behind that should not be" +ENVOUT="$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$(docker compose -f $DIR/compose.yaml ps -q server)")" +grep -q "RECOVERY_ADMIN" <<<"$ENVOUT" && bad "the server still carries a recovery admin" || ok "no recovery admin on the server" + +echo +echo "==> $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 1ffe5da..ac43d11 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -48,15 +48,16 @@ const ( // Result is what the operator is left holding. type Result struct { - Dir string - AdminUser string - AdminPass string - FirstUser string - FirstPass string - ZoneFile string - ConsoleURL string - WebmailURL string - ServerURL string + Dir string + AdminUser string + AdminPass string + FirstUser string + FirstPass string + ZoneFile string + Certificate string // what issued the mail server's certificate, when one arrived + ConsoleURL string + WebmailURL string + ServerURL string } // Log is how apply reports progress. The interface will draw ticks from it; @@ -130,6 +131,34 @@ func Run(ctx context.Context, p plan.Plan, log Log) (*Result, error) { } } + // A private ACME CA has to be trusted by two programs that never see each + // other's trust store: the mail server, which asks for its own + // certificate over HTTP-01, and Caddy, which asks for the front ends'. + // The server gets the image's own roots plus this one as a file it mounts + // over its bundle; Caddy gets the root on its own. + if o.ACMECARoot != "" { + root, err := os.ReadFile(o.ACMECARoot) + if err != nil { + return nil, fmt.Errorf("reading the ACME CA root: %w", err) + } + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, err + } + bundle, err := docker.SystemCABundle(ctx, stack.ServerImage) + if err != nil { + return nil, fmt.Errorf("reading the server image's CA bundle: %w", err) + } + if err := os.WriteFile(filepath.Join(dir, "ca-bundle.crt"), append(bundle, root...), 0o644); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "acme-ca-root.pem"), root, 0o644); err != nil { + return nil, err + } + stack.CABundle = true + stack.ACMECARoot = "acme-ca-root.pem" + } + stack.ACMEDirectory = o.ACMEDirectory + log.Step("writing the deployment into %s", dir) if _, err := compose.Write(dir, &stack); err != nil { return nil, err @@ -246,9 +275,21 @@ func Run(ctx context.Context, p plan.Plan, log Log) (*Result, error) { if !o.Local { log.Step("turning on certificates") - if _, err := srv.EnableACME(ctx, domainID, "", o.ACMEEmail); err != nil { + if _, err := srv.EnableACME(ctx, domainID, o.ACMEDirectory, o.ACMEEmail); err != nil { return res, fmt.Errorf("enabling ACME: %w", err) } + // An order that fails is not retried on its own, and a restart does + // not start a new one: moving the domain to manual and back is what + // does. So this waits, and asks again every so often, rather than + // declaring the install finished over a self-signed certificate that + // every mail client will refuse. + if issuer := waitForCertificate(ctx, srv, log, domainID, o.MailHost, 3*time.Minute); issuer != "" { + res.Certificate = issuer + log.Info("certificate for %s issued by %s", o.MailHost, issuer) + } else { + log.Info("no certificate yet for %s: the server keeps trying, and will succeed once %s resolves to this machine and port 80 reaches it", + o.MailHost, o.MailHost) + } } log.Step("creating the first account") @@ -276,6 +317,36 @@ func Run(ctx context.Context, p plan.Plan, log Log) (*Result, error) { return res, nil } +// waitForCertificate waits for the mail server's own certificate to arrive, +// asking for a fresh order every 45 seconds. It returns the issuer, or "" +// when none arrived in time -- which is not a failure: on a real install the +// domain often does not point here yet, and the server goes on trying. +func waitForCertificate(ctx context.Context, srv *jmap.Client, log Log, domainID, host string, limit time.Duration) string { + deadline := time.Now().Add(limit) + nextRetry := time.Now().Add(45 * time.Second) + for time.Now().Before(deadline) { + certs, err := srv.Certificates(ctx) + if err == nil { + for _, c := range certs { + if c.SubjectAlternativeNames[host] && !strings.Contains(c.Issuer, "self signed") { + return c.Issuer + } + } + } + if time.Now().After(nextRetry) { + log.Info("asking for the certificate again") + _ = srv.RetryCertificates(ctx, domainID) + nextRetry = time.Now().Add(45 * time.Second) + } + select { + case <-ctx.Done(): + return "" + case <-time.After(3 * time.Second): + } + } + return "" +} + // Verify is the last step: does what was installed actually answer? func Verify(ctx context.Context, res *Result, stackConsole, stackWebmail bool, log Log) []string { var problems []string diff --git a/internal/compose/compose.go b/internal/compose/compose.go index 8e4ac1e..f3e6be7 100644 --- a/internal/compose/compose.go +++ b/internal/compose/compose.go @@ -71,10 +71,17 @@ type Stack struct { CABundle bool } -// ServerNames is every name the server's certificate and the proxy need. +// ServerNames is every name the server's certificate and the proxy need: +// the mail host, and the four service names the server puts in its own +// certificate by default. +// +// The list has to match what the server asks for exactly. Leaving +// ua-auto-config out of the proxy meant its HTTP-01 challenge was not +// forwarded, the whole order failed on that one name, and the mail ports +// served a self-signed certificate while every other name validated. func (s Stack) ServerNames() []string { names := []string{s.MailHost} - for _, n := range []string{"autoconfig", "autodiscover", "mta-sts"} { + for _, n := range []string{"autoconfig", "autodiscover", "mta-sts", "ua-auto-config"} { names = append(names, n+"."+s.Domain) } return names diff --git a/internal/plan/plan.go b/internal/plan/plan.go index c1675ca..9884647 100644 --- a/internal/plan/plan.go +++ b/internal/plan/plan.go @@ -61,6 +61,12 @@ type Options struct { Proxy string // "caddy", "snippets", "none" Choices []Choice InstallDeps bool // resolve what is missing rather than refusing over it + + // A private ACME CA, for testing the certificate path without asking a + // public CA for certificates for names that are not ours. Empty means + // Let's Encrypt, which is what an install should use. + ACMEDirectory string + ACMECARoot string } // Shape returns what was chosen for a component.