From 2abab8264d982942fb7f122f6062d9046415b45c Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 15 Nov 2026 22:36:39 -0700 Subject: [PATCH] Certificates: critical only in the last 48 hours A certificate four days from its end was critical, so it repeated every 15 minutes at the same level as an outage. Now every certificate condition carries its end, and one pass holds it at warning until it's within certificates.critical_within (default 48h) of it, whichever source judged it critical: hosts' ports and files, site checks, Cloudflare, Akamai CPS, Vault and cert-manager. An ended one stays critical. The Certificates page judges by the same window; a certificate only in files still says nothing until 7 days out, and warns from there. Closes #34. --- docs/cert-inventory.md | 13 ++++--- docs/formats.md | 15 ++++++++ internal/server/akamai.go | 6 ++-- internal/server/akamai_test.go | 4 +-- internal/server/api.go | 4 +++ internal/server/certs.go | 62 +++++++++++++++++++++++++++------- internal/server/certs_test.go | 34 ++++++++++++++++--- internal/server/checks.go | 2 +- internal/server/cloudflare.go | 4 +-- internal/server/config.go | 8 +++++ internal/server/workloads.go | 3 +- 11 files changed, 125 insertions(+), 30 deletions(-) diff --git a/docs/cert-inventory.md b/docs/cert-inventory.md index 6d54ccf..d062e5a 100644 --- a/docs/cert-inventory.md +++ b/docs/cert-inventory.md @@ -78,16 +78,21 @@ where is it used?" ## What's judged -- **Ends:** 21 days a warning, 7 critical, as site checks and Vault; a +- **Ends:** a warning from 21 days, critical within the last 48 hours + (`certificates.critical_within` in obs-beacond.yaml, formats.md); a short-lived certificate (under 7 days' life, ACME's 6-day ones - included) by a third and a tenth of its life. + included) by a third and a tenth of its life. The window is the same + for every source's alerts: a source that judges a certificate critical + earlier (site checks and Vault at 7 days, Cloudflare at 3) pages only + inside it, and warns until then. Four days left is a job for working + hours; a page every 15 minutes for it gets the whole tool muted. - **Served but wrong** (from handshakes only): a name the certificate doesn't cover, an incomplete chain (missing intermediate), a self-signed certificate on a port that had a CA-signed one. Warnings. - **Weak** (question 6): SHA-1 signatures, RSA under 2048 bits. Warnings, or only shown. - **Not judged:** CA bundles, and certificates in files no service - serves, unless they end within 7 days (a renewed file nobody reloaded + serves, until 7 days before they end (a renewed file nobody reloaded is the classic outage: question 4). ## Scale @@ -162,7 +167,7 @@ modification time changes. `**` is any number of directories (eight at most) and `{a,b}` either; 5,000 files and 1 MiB a file at most. - **Places** of kind `file`, "node1:/etc/letsencrypt/live/shop/cert.pem". - A certificate only in files is judged at 7 days, not 21: a file no + A certificate only in files warns from 7 days, not 21: a file no port serves may be nobody's. Its alert says "no port serves it". - **Windows** (2026-11-15): a Windows agent has no helper and no files to glob; it reads the machine's certificate stores itself, diff --git a/docs/formats.md b/docs/formats.md index abac842..2e3c939 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -321,6 +321,21 @@ Without `alerts.channels` in obs-beacond.yaml, channels are added under Configure › Alerts instead (see "Settings edited in the UI"). Writing the key, even as `channels: []`, makes the file the place for them. +### Certificates + +`certificates.critical_within` (default `48h`) is how close to its end a +certificate's alert becomes critical, whichever source saw it: hosts' +ports and files, site checks, Cloudflare, Akamai CPS, Vault and the +clusters' cert-manager. Before that it's a warning, from when its source +starts warning (21 days for most; checks.yaml's `tls_warn_days` and +Cloudflare's `cert_warn_days` set theirs). A certificate that has ended is +critical. The Certificates page judges by the same window. + +```yaml +certificates: + critical_within: 72h +``` + ### State `database` is obs-beacond's own database: sessions, incidents (open ones and diff --git a/internal/server/akamai.go b/internal/server/akamai.go index d364da2..71aa498 100644 --- a/internal/server/akamai.go +++ b/internal/server/akamai.go @@ -79,8 +79,8 @@ func (srv *Server) akamaiConditions(now time.Time) []Condition { for _, e := range s.Enrollments { key := "akamai/cps/" + strconv.Itoa(e.ID) for _, c := range e.Deployed { - if st, word := certStatus(c, now); st != "good" { - out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, + if st, word := certStatus(c, now, srv.certCritical()); st != "good" { + out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, Ends: c.NotAfter, Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word), Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href}) } @@ -287,7 +287,7 @@ func (srv *Server) handleAkamai(w http.ResponseWriter, r *http.Request) { now := time.Now() for _, e := range snap.Enrollments { for _, c := range e.Deployed { - st, word := certStatus(c, now) + st, word := certStatus(c, now, srv.certCritical()) certs[c.Fingerprint] = judged{st, word, certinfo.Weak(&c)} } } diff --git a/internal/server/akamai_test.go b/internal/server/akamai_test.go index ad404d5..9787bad 100644 --- a/internal/server/akamai_test.go +++ b/internal/server/akamai_test.go @@ -92,7 +92,7 @@ func TestAkamaiCPS(t *testing.T) { srv := akServer(t, akamai.Snapshot{Checked: now, Domains: []akamai.Domain{}, Enrollments: []akamai.Enrollment{ {ID: 1001, CN: "www.example.com", Validation: "dv", Deployed: []report.Cert{cert("www.example.com", "aa", "RSA 2048", 60*day), cert("www.example.com", "ab", "ECDSA P-256", 60*day)}}, {ID: 1002, CN: "shop.example.com", Validation: "third-party", Pending: 1, Waiting: "third-party-csr", Deployed: []report.Cert{cert("shop.example.com", "bb", "RSA 2048", 10*day)}}, - {ID: 1003, CN: "api.example.com", Validation: "dv", Deployed: []report.Cert{cert("api.example.com", "cc", "RSA 2048", 3*day)}}, + {ID: 1003, CN: "api.example.com", Validation: "dv", Deployed: []report.Cert{cert("api.example.com", "cc", "RSA 2048", day)}}, {ID: 1004, CN: "new.example.com", Validation: "dv", Pending: 1, Deployed: []report.Cert{}}, }}) got := srv.akamaiConditions(now) @@ -100,7 +100,7 @@ func TestAkamaiCPS(t *testing.T) { t.Fatalf("%s", k) } if !strings.Contains(got[0].Detail, "third-party certificate") || !strings.Contains(got[1].Detail, "signed by your certificate authority") || - !strings.Contains(got[2].Detail, "stalled") || !strings.HasPrefix(got[2].Title, "Akamai certificate for api.example.com: ends in 3 days") { + !strings.Contains(got[2].Detail, "stalled") || !strings.HasPrefix(got[2].Title, "Akamai certificate for api.example.com: ends in ") || !strings.Contains(got[2].Title, " hours") { t.Errorf("%+v", got) } rows := srv.certificates(now) diff --git a/internal/server/api.go b/internal/server/api.go index 7d65a41..75420fe 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -217,6 +217,9 @@ type Condition struct { Affected string `json:"affected,omitempty"` // what else this takes down (root-cause grouping) Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it, when any did Map string `json:"map,omitempty"` // where it is on the maps: a page path with its focus + // Ends is when the certificate a condition is about ends, for one that + // is: certSeverity keeps it a warning until it's close (#34). + Ends time.Time `json:"-"` } // view judges every host now: what the UI shows, and the conditions the @@ -364,6 +367,7 @@ func (srv *Server) view(now time.Time) ([]HostView, []Condition) { conds = append(conds, dlConds...) conds = append(conds, srv.workloadConditions(now)...) conds = append(conds, srv.certConditions(now)...) + certSeverity(conds, now, srv.certCritical()) if byHost, _ := srv.vmLinks(); len(byHost) > 0 { explainSilent(conds, byHost) for i := range hosts { diff --git a/internal/server/certs.go b/internal/server/certs.go index c4495ef..74f5e35 100644 --- a/internal/server/certs.go +++ b/internal/server/certs.go @@ -22,11 +22,43 @@ import ( // with each source (decision 5); this page only gathers. const ( - certWarn = 21 * 24 * time.Hour - certCritical = 7 * 24 * time.Hour - certPlaces = 50 // places sent per certificate; PlaceCount has them all + certWarn = 21 * 24 * time.Hour + certPlaces = 50 // places sent per certificate; PlaceCount has them all + // defaultCertCritical is certificates.critical_within's default: four + // days left is a job for working hours, two is not (#34). + defaultCertCritical = 48 * time.Hour ) +// CertificatesConfig is obs-beacond.yaml's certificates section. +type CertificatesConfig struct { + // CriticalWithin is how close to its end a certificate becomes + // critical (48h). Before that it's a warning, from whenever its source + // starts warning (21 days for most). + CriticalWithin time.Duration `yaml:"critical_within"` +} + +func (srv *Server) certCritical() time.Duration { + if srv.cfg == nil || srv.cfg.Certificates.CriticalWithin <= 0 { + return defaultCertCritical + } + return srv.cfg.Certificates.CriticalWithin +} + +// certSeverity holds every certificate condition at warning until its +// certificate is within critical of its end, whatever its source judged +// (each has its own critical threshold: 7 days for checks and Vault, 3 +// for Cloudflare). A source's critical inside the window stands, and so +// does one about something other than the end (an ended certificate is +// past it). +func certSeverity(conds []Condition, now time.Time, critical time.Duration) { + for i := range conds { + c := &conds[i] + if c.Severity == "critical" && !c.Ends.IsZero() && c.Ends.Sub(now) > critical { + c.Severity = "warning" + } + } +} + // CertInvPlace is where a certificate was seen. type CertInvPlace struct { Kind string `json:"kind"` // "host", "file", "store", "vault", "check", "cloudflare", "cluster", "akamai" @@ -88,11 +120,12 @@ type CertReload struct { // reload before it's said not to have happened. const reloadGrace = time.Hour -// certStatus judges a certificate by the time it has left: 21 and 7 days, -// or a third and a tenth of a short life (the same as Vault's leaves). -func certStatus(c report.Cert, now time.Time) (string, string) { +// certStatus judges a certificate by the time it has left: 21 days, and +// critical (certificates.critical_within), or a third and a tenth of a +// short life (the same as Vault's leaves). +func certStatus(c report.Cert, now time.Time, critical time.Duration) (string, string) { left, life := c.NotAfter.Sub(now), c.NotAfter.Sub(c.NotBefore) - warn, crit := certWarn, certCritical + warn, crit := certWarn, critical if life > 0 && !c.CA { warn, crit = min(warn, life/3), min(crit, life/10) } @@ -296,9 +329,9 @@ func (srv *Server) certificates(now time.Time) []*CertInv { if row.NotAfter.Before(now) { row.Days = -int(now.Sub(row.NotAfter).Hours()/24) - 1 } - row.Status, row.Word = certStatus(row.Cert, now) - if fileOnly(row) && row.Status == "warning" { - row.Status = "good" // a file no port serves is judged at 7 days only + row.Status, row.Word = certStatus(row.Cert, now, srv.certCritical()) + if fileOnly(row) && row.Status == "warning" && row.NotAfter.Sub(now) >= fileWarn { + row.Status = "good" // a file no port serves is judged from 7 days only } if row.Weak = certinfo.Weak(&row.Cert); row.Weak != "" && row.Status == "good" && !fileOnly(row) { row.Status = "warning" @@ -373,9 +406,12 @@ func namesKey(c report.Cert) string { return strings.Join(c.Names, ",") } +// fileWarn is when a certificate found only in files starts to warn: 7 +// days, not 21 (one no port serves may be nobody's). +const fileWarn = 7 * 24 * time.Hour + // fileOnly says a certificate was found only in files or a Windows -// certificate store: judged at 7 days, not 21 (one no port serves may be -// nobody's). +// certificate store. func fileOnly(c *CertInv) bool { for _, k := range c.Kinds { if k != "file" && k != "store" { @@ -528,7 +564,7 @@ func (srv *Server) certConditions(now time.Time) []Condition { if !own { sev = "warning" // the others alert on its end; this is only the reload } - cond := Condition{Key: "certinv/" + c.Fingerprint, Severity: sev, Title: "Certificate " + c.Subject + " " + what, + cond := Condition{Key: "certinv/" + c.Fingerprint, Severity: sev, Ends: c.NotAfter, Title: "Certificate " + c.Subject + " " + what, Detail: strings.Join(detail, " "), Href: "/certificates?q=" + c.Fingerprint[:16]} if len(hosts) == 1 { for h := range hosts { diff --git a/internal/server/certs_test.go b/internal/server/certs_test.go index 91843c3..d10b882 100644 --- a/internal/server/certs_test.go +++ b/internal/server/certs_test.go @@ -66,7 +66,7 @@ func TestCertificates(t *testing.T) { srv.store.Put("node1", results(now.Add(time.Second)), now.Add(time.Second)) // kept between scans conds := srv.certConditions(now) - if len(conds) != 1 || conds[0].Key != "certinv/"+mine.Fingerprint || conds[0].Severity != "critical" || conds[0].Host != "node1" || + if len(conds) != 1 || conds[0].Key != "certinv/"+mine.Fingerprint || conds[0].Severity != "warning" || conds[0].Host != "node1" || // 4 days: a warning (#34) conds[0].Detail != "Served on node1:8443 · admin, node1:25 (smtp STARTTLS) · postfix." { t.Errorf("conditions: %+v", conds) } @@ -136,7 +136,7 @@ func TestCertFiles(t *testing.T) { for _, c := range srv.certificates(now) { rows[c.Fingerprint] = c } - if c := rows[old.Fingerprint]; c == nil || c.Status != "critical" || len(c.NotReloaded) != 1 || c.NotReloaded[0] != "node1:8443 · admin" || + if c := rows[old.Fingerprint]; c == nil || c.Status != "warning" || len(c.NotReloaded) != 1 || c.NotReloaded[0] != "node1:8443 · admin" || !strings.HasPrefix(c.Renewed, "node1:/etc/letsencrypt/live/admin/cert.pem (ends ") { t.Errorf("the admin's: %+v", c) } @@ -154,14 +154,14 @@ func TestCertFiles(t *testing.T) { if len(conds) != 3 { t.Errorf("%d conditions: %+v", len(conds), conds) } - if c := conds["certinv/"+old.Fingerprint]; c.Severity != "critical" || c.Host != "node1" || !strings.HasSuffix(c.Title, "; renewed but not reloaded") || + if c := conds["certinv/"+old.Fingerprint]; c.Severity != "warning" || c.Host != "node1" || !strings.HasSuffix(c.Title, "; renewed but not reloaded") || !strings.Contains(c.Detail, "but node1:8443 · admin still serves this one: reload the service.") { t.Errorf("the admin's: %+v", c) } if c := conds["certinv/"+shop.Fingerprint]; c.Severity != "warning" || c.Title != "Certificate shop.example.org renewed but not reloaded" { t.Errorf("the shop's, which a check alerts on: %+v", c) } - if c := conds["certinv/"+ending.Fingerprint]; c.Severity != "critical" || c.Detail != "In node1:/etc/pki/tls/certs/ending.crt; no port serves it." { + if c := conds["certinv/"+ending.Fingerprint]; c.Severity != "warning" || c.Detail != "In node1:/etc/pki/tls/certs/ending.crt; no port serves it." { t.Errorf("a file ending in 3 days: %+v", c) } @@ -264,3 +264,29 @@ func TestCertWrong(t *testing.T) { } } } + +// A certificate's alert is a warning until it's within critical_within of +// its end, whichever source judged it critical (#34). +func TestCertSeverity(t *testing.T) { + now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC) + conds := []Condition{ + {Key: "four days", Severity: "critical", Ends: now.Add(4 * 24 * time.Hour)}, + {Key: "a day", Severity: "critical", Ends: now.Add(24 * time.Hour)}, + {Key: "ended", Severity: "critical", Ends: now.Add(-time.Hour)}, + {Key: "not a certificate", Severity: "critical"}, + {Key: "warning", Severity: "warning", Ends: now.Add(10 * 24 * time.Hour)}, + } + certSeverity(conds, now, defaultCertCritical) + want := []string{"warning", "critical", "critical", "critical", "warning"} + for i, c := range conds { + if c.Severity != want[i] { + t.Errorf("%s: %s, want %s", c.Key, c.Severity, want[i]) + } + } + if st, _ := certStatus(report.Cert{NotBefore: now.Add(-300 * 24 * time.Hour), NotAfter: now.Add(4 * 24 * time.Hour)}, now, defaultCertCritical); st != "warning" { + t.Errorf("certStatus four days out: %s", st) + } + if st, _ := certStatus(report.Cert{NotBefore: now.Add(-300 * 24 * time.Hour), NotAfter: now.Add(4 * 24 * time.Hour)}, now, 7*24*time.Hour); st != "critical" { + t.Errorf("certStatus four days out, critical_within 7d: %s", st) + } +} diff --git a/internal/server/checks.go b/internal/server/checks.go index f8d63a0..bddf49a 100644 --- a/internal/server/checks.go +++ b/internal/server/checks.go @@ -269,7 +269,7 @@ func (srv *Server) judgeChecks(now time.Time) ([]CheckView, []Condition) { sev = "warning" } if sev != "" { - conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, + conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, Ends: soonest, Title: fmt.Sprintf("%s certificate expires in %s", s.Name, plural(days, "day", "days")), Detail: fmt.Sprintf("Seen on %s; it expires %s.", s.Target(), soonest.UTC().Format("2006-01-02 15:04 UTC")), Href: href}) if v.Status == "good" { diff --git a/internal/server/cloudflare.go b/internal/server/cloudflare.go index 3eee1f9..6e95fa6 100644 --- a/internal/server/cloudflare.go +++ b/internal/server/cloudflare.go @@ -108,11 +108,11 @@ func (srv *Server) cloudflareConditions(now time.Time) []Condition { Detail: "Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href}) case c.Expires.IsZero(): case days < cfg.CertCriticalDays: - out = append(out, Condition{Key: key, Severity: "critical", + out = append(out, Condition{Key: key, Severity: "critical", Ends: c.Expires, Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")), Detail: "Cloudflare normally renews well before this; renewal is stuck. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href}) case days < cfg.CertWarnDays: - out = append(out, Condition{Key: key, Severity: "warning", + out = append(out, Condition{Key: key, Severity: "warning", Ends: c.Expires, Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")), Detail: "Cloudflare normally renews 30 days ahead. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href}) } diff --git a/internal/server/config.go b/internal/server/config.go index 6d9e718..e5d2424 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -74,6 +74,8 @@ type Config struct { Mail *mailmon.Config `yaml:"mail"` // Reports' timezone and the weekly digest. Reports *ReportsConfig `yaml:"reports"` + // How close to its end a certificate becomes critical, from any source. + Certificates CertificatesConfig `yaml:"certificates"` // Vault and OpenBao clusters obs-beacond reads itself (docs/vault.md). Vault []vault.Config `yaml:"vault"` } @@ -142,6 +144,12 @@ func LoadConfig(p string) (*Config, error) { if c.StaleAfter == 0 { c.StaleAfter = 60 * time.Second } + if c.Certificates.CriticalWithin < 0 { + return nil, fmt.Errorf("%s: certificates.critical_within can't be negative", p) + } + if c.Certificates.CriticalWithin == 0 { + c.Certificates.CriticalWithin = defaultCertCritical + } if c.EmergencyListen != "" && !loopback(c.EmergencyListen) { return nil, fmt.Errorf("%s: emergency_listen must be a loopback address (it's reached through an SSH tunnel), not %s", p, c.EmergencyListen) } diff --git a/internal/server/workloads.go b/internal/server/workloads.go index d512eae..e6ee9d0 100644 --- a/internal/server/workloads.go +++ b/internal/server/workloads.go @@ -855,7 +855,8 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt if on, _ := r.Facts["served_on"].(string); on != "" { detail = append(detail, "Served on "+on+".") } - add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), + ends, _ := time.Parse(time.RFC3339, fmt.Sprint(r.Facts["ends"])) + add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends, Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind) } case report.KindVM: -- 2.54.0