diff --git a/docs/cert-inventory.md b/docs/cert-inventory.md index 6d54ccf..d062e5a 100644 --- a/docs/cert-inventory.md +++ b/docs/cert-inventory.md @@ -78,16 +78,21 @@ where is it used?" ## What's judged -- **Ends:** 21 days a warning, 7 critical, as site checks and Vault; a +- **Ends:** a warning from 21 days, critical within the last 48 hours + (`certificates.critical_within` in obs-beacond.yaml, formats.md); a short-lived certificate (under 7 days' life, ACME's 6-day ones - included) by a third and a tenth of its life. + included) by a third and a tenth of its life. The window is the same + for every source's alerts: a source that judges a certificate critical + earlier (site checks and Vault at 7 days, Cloudflare at 3) pages only + inside it, and warns until then. Four days left is a job for working + hours; a page every 15 minutes for it gets the whole tool muted. - **Served but wrong** (from handshakes only): a name the certificate doesn't cover, an incomplete chain (missing intermediate), a self-signed certificate on a port that had a CA-signed one. Warnings. - **Weak** (question 6): SHA-1 signatures, RSA under 2048 bits. Warnings, or only shown. - **Not judged:** CA bundles, and certificates in files no service - serves, unless they end within 7 days (a renewed file nobody reloaded + serves, until 7 days before they end (a renewed file nobody reloaded is the classic outage: question 4). ## Scale @@ -162,7 +167,7 @@ modification time changes. `**` is any number of directories (eight at most) and `{a,b}` either; 5,000 files and 1 MiB a file at most. - **Places** of kind `file`, "node1:/etc/letsencrypt/live/shop/cert.pem". - A certificate only in files is judged at 7 days, not 21: a file no + A certificate only in files warns from 7 days, not 21: a file no port serves may be nobody's. Its alert says "no port serves it". - **Windows** (2026-11-15): a Windows agent has no helper and no files to glob; it reads the machine's certificate stores itself, diff --git a/docs/formats.md b/docs/formats.md index abac842..2e3c939 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -321,6 +321,21 @@ Without `alerts.channels` in obs-beacond.yaml, channels are added under Configure › Alerts instead (see "Settings edited in the UI"). Writing the key, even as `channels: []`, makes the file the place for them. +### Certificates + +`certificates.critical_within` (default `48h`) is how close to its end a +certificate's alert becomes critical, whichever source saw it: hosts' +ports and files, site checks, Cloudflare, Akamai CPS, Vault and the +clusters' cert-manager. Before that it's a warning, from when its source +starts warning (21 days for most; checks.yaml's `tls_warn_days` and +Cloudflare's `cert_warn_days` set theirs). A certificate that has ended is +critical. The Certificates page judges by the same window. + +```yaml +certificates: + critical_within: 72h +``` + ### State `database` is obs-beacond's own database: sessions, incidents (open ones and diff --git a/internal/server/akamai.go b/internal/server/akamai.go index d364da2..71aa498 100644 --- a/internal/server/akamai.go +++ b/internal/server/akamai.go @@ -79,8 +79,8 @@ func (srv *Server) akamaiConditions(now time.Time) []Condition { for _, e := range s.Enrollments { key := "akamai/cps/" + strconv.Itoa(e.ID) for _, c := range e.Deployed { - if st, word := certStatus(c, now); st != "good" { - out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, + if st, word := certStatus(c, now, srv.certCritical()); st != "good" { + out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, Ends: c.NotAfter, Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word), Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href}) } @@ -287,7 +287,7 @@ func (srv *Server) handleAkamai(w http.ResponseWriter, r *http.Request) { now := time.Now() for _, e := range snap.Enrollments { for _, c := range e.Deployed { - st, word := certStatus(c, now) + st, word := certStatus(c, now, srv.certCritical()) certs[c.Fingerprint] = judged{st, word, certinfo.Weak(&c)} } } diff --git a/internal/server/akamai_test.go b/internal/server/akamai_test.go index ad404d5..9787bad 100644 --- a/internal/server/akamai_test.go +++ b/internal/server/akamai_test.go @@ -92,7 +92,7 @@ func TestAkamaiCPS(t *testing.T) { srv := akServer(t, akamai.Snapshot{Checked: now, Domains: []akamai.Domain{}, Enrollments: []akamai.Enrollment{ {ID: 1001, CN: "www.example.com", Validation: "dv", Deployed: []report.Cert{cert("www.example.com", "aa", "RSA 2048", 60*day), cert("www.example.com", "ab", "ECDSA P-256", 60*day)}}, {ID: 1002, CN: "shop.example.com", Validation: "third-party", Pending: 1, Waiting: "third-party-csr", Deployed: []report.Cert{cert("shop.example.com", "bb", "RSA 2048", 10*day)}}, - {ID: 1003, CN: "api.example.com", Validation: "dv", Deployed: []report.Cert{cert("api.example.com", "cc", "RSA 2048", 3*day)}}, + {ID: 1003, CN: "api.example.com", Validation: "dv", Deployed: []report.Cert{cert("api.example.com", "cc", "RSA 2048", day)}}, {ID: 1004, CN: "new.example.com", Validation: "dv", Pending: 1, Deployed: []report.Cert{}}, }}) got := srv.akamaiConditions(now) @@ -100,7 +100,7 @@ func TestAkamaiCPS(t *testing.T) { t.Fatalf("%s", k) } if !strings.Contains(got[0].Detail, "third-party certificate") || !strings.Contains(got[1].Detail, "signed by your certificate authority") || - !strings.Contains(got[2].Detail, "stalled") || !strings.HasPrefix(got[2].Title, "Akamai certificate for api.example.com: ends in 3 days") { + !strings.Contains(got[2].Detail, "stalled") || !strings.HasPrefix(got[2].Title, "Akamai certificate for api.example.com: ends in ") || !strings.Contains(got[2].Title, " hours") { t.Errorf("%+v", got) } rows := srv.certificates(now) diff --git a/internal/server/api.go b/internal/server/api.go index 7d65a41..75420fe 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -217,6 +217,9 @@ type Condition struct { Affected string `json:"affected,omitempty"` // what else this takes down (root-cause grouping) Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it, when any did Map string `json:"map,omitempty"` // where it is on the maps: a page path with its focus + // Ends is when the certificate a condition is about ends, for one that + // is: certSeverity keeps it a warning until it's close (#34). + Ends time.Time `json:"-"` } // view judges every host now: what the UI shows, and the conditions the @@ -364,6 +367,7 @@ func (srv *Server) view(now time.Time) ([]HostView, []Condition) { conds = append(conds, dlConds...) conds = append(conds, srv.workloadConditions(now)...) conds = append(conds, srv.certConditions(now)...) + certSeverity(conds, now, srv.certCritical()) if byHost, _ := srv.vmLinks(); len(byHost) > 0 { explainSilent(conds, byHost) for i := range hosts { diff --git a/internal/server/certs.go b/internal/server/certs.go index c4495ef..74f5e35 100644 --- a/internal/server/certs.go +++ b/internal/server/certs.go @@ -22,11 +22,43 @@ import ( // with each source (decision 5); this page only gathers. const ( - certWarn = 21 * 24 * time.Hour - certCritical = 7 * 24 * time.Hour - certPlaces = 50 // places sent per certificate; PlaceCount has them all + certWarn = 21 * 24 * time.Hour + certPlaces = 50 // places sent per certificate; PlaceCount has them all + // defaultCertCritical is certificates.critical_within's default: four + // days left is a job for working hours, two is not (#34). + defaultCertCritical = 48 * time.Hour ) +// CertificatesConfig is obs-beacond.yaml's certificates section. +type CertificatesConfig struct { + // CriticalWithin is how close to its end a certificate becomes + // critical (48h). Before that it's a warning, from whenever its source + // starts warning (21 days for most). + CriticalWithin time.Duration `yaml:"critical_within"` +} + +func (srv *Server) certCritical() time.Duration { + if srv.cfg == nil || srv.cfg.Certificates.CriticalWithin <= 0 { + return defaultCertCritical + } + return srv.cfg.Certificates.CriticalWithin +} + +// certSeverity holds every certificate condition at warning until its +// certificate is within critical of its end, whatever its source judged +// (each has its own critical threshold: 7 days for checks and Vault, 3 +// for Cloudflare). A source's critical inside the window stands, and so +// does one about something other than the end (an ended certificate is +// past it). +func certSeverity(conds []Condition, now time.Time, critical time.Duration) { + for i := range conds { + c := &conds[i] + if c.Severity == "critical" && !c.Ends.IsZero() && c.Ends.Sub(now) > critical { + c.Severity = "warning" + } + } +} + // CertInvPlace is where a certificate was seen. type CertInvPlace struct { Kind string `json:"kind"` // "host", "file", "store", "vault", "check", "cloudflare", "cluster", "akamai" @@ -88,11 +120,12 @@ type CertReload struct { // reload before it's said not to have happened. const reloadGrace = time.Hour -// certStatus judges a certificate by the time it has left: 21 and 7 days, -// or a third and a tenth of a short life (the same as Vault's leaves). -func certStatus(c report.Cert, now time.Time) (string, string) { +// certStatus judges a certificate by the time it has left: 21 days, and +// critical (certificates.critical_within), or a third and a tenth of a +// short life (the same as Vault's leaves). +func certStatus(c report.Cert, now time.Time, critical time.Duration) (string, string) { left, life := c.NotAfter.Sub(now), c.NotAfter.Sub(c.NotBefore) - warn, crit := certWarn, certCritical + warn, crit := certWarn, critical if life > 0 && !c.CA { warn, crit = min(warn, life/3), min(crit, life/10) } @@ -296,9 +329,9 @@ func (srv *Server) certificates(now time.Time) []*CertInv { if row.NotAfter.Before(now) { row.Days = -int(now.Sub(row.NotAfter).Hours()/24) - 1 } - row.Status, row.Word = certStatus(row.Cert, now) - if fileOnly(row) && row.Status == "warning" { - row.Status = "good" // a file no port serves is judged at 7 days only + row.Status, row.Word = certStatus(row.Cert, now, srv.certCritical()) + if fileOnly(row) && row.Status == "warning" && row.NotAfter.Sub(now) >= fileWarn { + row.Status = "good" // a file no port serves is judged from 7 days only } if row.Weak = certinfo.Weak(&row.Cert); row.Weak != "" && row.Status == "good" && !fileOnly(row) { row.Status = "warning" @@ -373,9 +406,12 @@ func namesKey(c report.Cert) string { return strings.Join(c.Names, ",") } +// fileWarn is when a certificate found only in files starts to warn: 7 +// days, not 21 (one no port serves may be nobody's). +const fileWarn = 7 * 24 * time.Hour + // fileOnly says a certificate was found only in files or a Windows -// certificate store: judged at 7 days, not 21 (one no port serves may be -// nobody's). +// certificate store. func fileOnly(c *CertInv) bool { for _, k := range c.Kinds { if k != "file" && k != "store" { @@ -528,7 +564,7 @@ func (srv *Server) certConditions(now time.Time) []Condition { if !own { sev = "warning" // the others alert on its end; this is only the reload } - cond := Condition{Key: "certinv/" + c.Fingerprint, Severity: sev, Title: "Certificate " + c.Subject + " " + what, + cond := Condition{Key: "certinv/" + c.Fingerprint, Severity: sev, Ends: c.NotAfter, Title: "Certificate " + c.Subject + " " + what, Detail: strings.Join(detail, " "), Href: "/certificates?q=" + c.Fingerprint[:16]} if len(hosts) == 1 { for h := range hosts { diff --git a/internal/server/certs_test.go b/internal/server/certs_test.go index 91843c3..d10b882 100644 --- a/internal/server/certs_test.go +++ b/internal/server/certs_test.go @@ -66,7 +66,7 @@ func TestCertificates(t *testing.T) { srv.store.Put("node1", results(now.Add(time.Second)), now.Add(time.Second)) // kept between scans conds := srv.certConditions(now) - if len(conds) != 1 || conds[0].Key != "certinv/"+mine.Fingerprint || conds[0].Severity != "critical" || conds[0].Host != "node1" || + if len(conds) != 1 || conds[0].Key != "certinv/"+mine.Fingerprint || conds[0].Severity != "warning" || conds[0].Host != "node1" || // 4 days: a warning (#34) conds[0].Detail != "Served on node1:8443 · admin, node1:25 (smtp STARTTLS) · postfix." { t.Errorf("conditions: %+v", conds) } @@ -136,7 +136,7 @@ func TestCertFiles(t *testing.T) { for _, c := range srv.certificates(now) { rows[c.Fingerprint] = c } - if c := rows[old.Fingerprint]; c == nil || c.Status != "critical" || len(c.NotReloaded) != 1 || c.NotReloaded[0] != "node1:8443 · admin" || + if c := rows[old.Fingerprint]; c == nil || c.Status != "warning" || len(c.NotReloaded) != 1 || c.NotReloaded[0] != "node1:8443 · admin" || !strings.HasPrefix(c.Renewed, "node1:/etc/letsencrypt/live/admin/cert.pem (ends ") { t.Errorf("the admin's: %+v", c) } @@ -154,14 +154,14 @@ func TestCertFiles(t *testing.T) { if len(conds) != 3 { t.Errorf("%d conditions: %+v", len(conds), conds) } - if c := conds["certinv/"+old.Fingerprint]; c.Severity != "critical" || c.Host != "node1" || !strings.HasSuffix(c.Title, "; renewed but not reloaded") || + if c := conds["certinv/"+old.Fingerprint]; c.Severity != "warning" || c.Host != "node1" || !strings.HasSuffix(c.Title, "; renewed but not reloaded") || !strings.Contains(c.Detail, "but node1:8443 · admin still serves this one: reload the service.") { t.Errorf("the admin's: %+v", c) } if c := conds["certinv/"+shop.Fingerprint]; c.Severity != "warning" || c.Title != "Certificate shop.example.org renewed but not reloaded" { t.Errorf("the shop's, which a check alerts on: %+v", c) } - if c := conds["certinv/"+ending.Fingerprint]; c.Severity != "critical" || c.Detail != "In node1:/etc/pki/tls/certs/ending.crt; no port serves it." { + if c := conds["certinv/"+ending.Fingerprint]; c.Severity != "warning" || c.Detail != "In node1:/etc/pki/tls/certs/ending.crt; no port serves it." { t.Errorf("a file ending in 3 days: %+v", c) } @@ -264,3 +264,29 @@ func TestCertWrong(t *testing.T) { } } } + +// A certificate's alert is a warning until it's within critical_within of +// its end, whichever source judged it critical (#34). +func TestCertSeverity(t *testing.T) { + now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC) + conds := []Condition{ + {Key: "four days", Severity: "critical", Ends: now.Add(4 * 24 * time.Hour)}, + {Key: "a day", Severity: "critical", Ends: now.Add(24 * time.Hour)}, + {Key: "ended", Severity: "critical", Ends: now.Add(-time.Hour)}, + {Key: "not a certificate", Severity: "critical"}, + {Key: "warning", Severity: "warning", Ends: now.Add(10 * 24 * time.Hour)}, + } + certSeverity(conds, now, defaultCertCritical) + want := []string{"warning", "critical", "critical", "critical", "warning"} + for i, c := range conds { + if c.Severity != want[i] { + t.Errorf("%s: %s, want %s", c.Key, c.Severity, want[i]) + } + } + if st, _ := certStatus(report.Cert{NotBefore: now.Add(-300 * 24 * time.Hour), NotAfter: now.Add(4 * 24 * time.Hour)}, now, defaultCertCritical); st != "warning" { + t.Errorf("certStatus four days out: %s", st) + } + if st, _ := certStatus(report.Cert{NotBefore: now.Add(-300 * 24 * time.Hour), NotAfter: now.Add(4 * 24 * time.Hour)}, now, 7*24*time.Hour); st != "critical" { + t.Errorf("certStatus four days out, critical_within 7d: %s", st) + } +} diff --git a/internal/server/checks.go b/internal/server/checks.go index f8d63a0..bddf49a 100644 --- a/internal/server/checks.go +++ b/internal/server/checks.go @@ -269,7 +269,7 @@ func (srv *Server) judgeChecks(now time.Time) ([]CheckView, []Condition) { sev = "warning" } if sev != "" { - conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, + conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, Ends: soonest, Title: fmt.Sprintf("%s certificate expires in %s", s.Name, plural(days, "day", "days")), Detail: fmt.Sprintf("Seen on %s; it expires %s.", s.Target(), soonest.UTC().Format("2006-01-02 15:04 UTC")), Href: href}) if v.Status == "good" { diff --git a/internal/server/cloudflare.go b/internal/server/cloudflare.go index 3eee1f9..6e95fa6 100644 --- a/internal/server/cloudflare.go +++ b/internal/server/cloudflare.go @@ -108,11 +108,11 @@ func (srv *Server) cloudflareConditions(now time.Time) []Condition { Detail: "Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href}) case c.Expires.IsZero(): case days < cfg.CertCriticalDays: - out = append(out, Condition{Key: key, Severity: "critical", + out = append(out, Condition{Key: key, Severity: "critical", Ends: c.Expires, Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")), Detail: "Cloudflare normally renews well before this; renewal is stuck. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href}) case days < cfg.CertWarnDays: - out = append(out, Condition{Key: key, Severity: "warning", + out = append(out, Condition{Key: key, Severity: "warning", Ends: c.Expires, Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")), Detail: "Cloudflare normally renews 30 days ahead. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href}) } diff --git a/internal/server/config.go b/internal/server/config.go index 6d9e718..e5d2424 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -74,6 +74,8 @@ type Config struct { Mail *mailmon.Config `yaml:"mail"` // Reports' timezone and the weekly digest. Reports *ReportsConfig `yaml:"reports"` + // How close to its end a certificate becomes critical, from any source. + Certificates CertificatesConfig `yaml:"certificates"` // Vault and OpenBao clusters obs-beacond reads itself (docs/vault.md). Vault []vault.Config `yaml:"vault"` } @@ -142,6 +144,12 @@ func LoadConfig(p string) (*Config, error) { if c.StaleAfter == 0 { c.StaleAfter = 60 * time.Second } + if c.Certificates.CriticalWithin < 0 { + return nil, fmt.Errorf("%s: certificates.critical_within can't be negative", p) + } + if c.Certificates.CriticalWithin == 0 { + c.Certificates.CriticalWithin = defaultCertCritical + } if c.EmergencyListen != "" && !loopback(c.EmergencyListen) { return nil, fmt.Errorf("%s: emergency_listen must be a loopback address (it's reached through an SSH tunnel), not %s", p, c.EmergencyListen) } diff --git a/internal/server/workloads.go b/internal/server/workloads.go index d512eae..e6ee9d0 100644 --- a/internal/server/workloads.go +++ b/internal/server/workloads.go @@ -855,7 +855,8 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt if on, _ := r.Facts["served_on"].(string); on != "" { detail = append(detail, "Served on "+on+".") } - add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), + ends, _ := time.Parse(time.RFC3339, fmt.Sprint(r.Facts["ends"])) + add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends, Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind) } case report.KindVM: