Certificates: critical only in the last 48 hours #49
No files matched your search
@@ -78,16 +78,21 @@ where is it used?"
|
||||
|
||||
## What's judged
|
||||
|
||||
- **Ends:** 21 days a warning, 7 critical, as site checks and Vault; a
|
||||
- **Ends:** a warning from 21 days, critical within the last 48 hours
|
||||
(`certificates.critical_within` in obs-beacond.yaml, formats.md); a
|
||||
short-lived certificate (under 7 days' life, ACME's 6-day ones
|
||||
included) by a third and a tenth of its life.
|
||||
included) by a third and a tenth of its life. The window is the same
|
||||
for every source's alerts: a source that judges a certificate critical
|
||||
earlier (site checks and Vault at 7 days, Cloudflare at 3) pages only
|
||||
inside it, and warns until then. Four days left is a job for working
|
||||
hours; a page every 15 minutes for it gets the whole tool muted.
|
||||
- **Served but wrong** (from handshakes only): a name the certificate
|
||||
doesn't cover, an incomplete chain (missing intermediate), a
|
||||
self-signed certificate on a port that had a CA-signed one. Warnings.
|
||||
- **Weak** (question 6): SHA-1 signatures, RSA under 2048 bits.
|
||||
Warnings, or only shown.
|
||||
- **Not judged:** CA bundles, and certificates in files no service
|
||||
serves, unless they end within 7 days (a renewed file nobody reloaded
|
||||
serves, until 7 days before they end (a renewed file nobody reloaded
|
||||
is the classic outage: question 4).
|
||||
|
||||
## Scale
|
||||
@@ -162,7 +167,7 @@ modification time changes.
|
||||
`**` is any number of directories (eight at most) and `{a,b}` either;
|
||||
5,000 files and 1 MiB a file at most.
|
||||
- **Places** of kind `file`, "node1:/etc/letsencrypt/live/shop/cert.pem".
|
||||
A certificate only in files is judged at 7 days, not 21: a file no
|
||||
A certificate only in files warns from 7 days, not 21: a file no
|
||||
port serves may be nobody's. Its alert says "no port serves it".
|
||||
- **Windows** (2026-11-15): a Windows agent has no helper and no
|
||||
files to glob; it reads the machine's certificate stores itself,
|
||||
|
||||
@@ -321,6 +321,21 @@ Without `alerts.channels` in obs-beacond.yaml, channels are added under
|
||||
Configure › Alerts instead (see "Settings edited in the UI"). Writing the key,
|
||||
even as `channels: []`, makes the file the place for them.
|
||||
|
||||
### Certificates
|
||||
|
||||
`certificates.critical_within` (default `48h`) is how close to its end a
|
||||
certificate's alert becomes critical, whichever source saw it: hosts'
|
||||
ports and files, site checks, Cloudflare, Akamai CPS, Vault and the
|
||||
clusters' cert-manager. Before that it's a warning, from when its source
|
||||
starts warning (21 days for most; checks.yaml's `tls_warn_days` and
|
||||
Cloudflare's `cert_warn_days` set theirs). A certificate that has ended is
|
||||
critical. The Certificates page judges by the same window.
|
||||
|
||||
```yaml
|
||||
certificates:
|
||||
critical_within: 72h
|
||||
```
|
||||
|
||||
### State
|
||||
|
||||
`database` is obs-beacond's own database: sessions, incidents (open ones and
|
||||
|
||||
@@ -79,8 +79,8 @@ func (srv *Server) akamaiConditions(now time.Time) []Condition {
|
||||
for _, e := range s.Enrollments {
|
||||
key := "akamai/cps/" + strconv.Itoa(e.ID)
|
||||
for _, c := range e.Deployed {
|
||||
if st, word := certStatus(c, now); st != "good" {
|
||||
out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st,
|
||||
if st, word := certStatus(c, now, srv.certCritical()); st != "good" {
|
||||
out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, Ends: c.NotAfter,
|
||||
Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word),
|
||||
Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href})
|
||||
}
|
||||
@@ -287,7 +287,7 @@ func (srv *Server) handleAkamai(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
for _, e := range snap.Enrollments {
|
||||
for _, c := range e.Deployed {
|
||||
st, word := certStatus(c, now)
|
||||
st, word := certStatus(c, now, srv.certCritical())
|
||||
certs[c.Fingerprint] = judged{st, word, certinfo.Weak(&c)}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestAkamaiCPS(t *testing.T) {
|
||||
srv := akServer(t, akamai.Snapshot{Checked: now, Domains: []akamai.Domain{}, Enrollments: []akamai.Enrollment{
|
||||
{ID: 1001, CN: "www.example.com", Validation: "dv", Deployed: []report.Cert{cert("www.example.com", "aa", "RSA 2048", 60*day), cert("www.example.com", "ab", "ECDSA P-256", 60*day)}},
|
||||
{ID: 1002, CN: "shop.example.com", Validation: "third-party", Pending: 1, Waiting: "third-party-csr", Deployed: []report.Cert{cert("shop.example.com", "bb", "RSA 2048", 10*day)}},
|
||||
{ID: 1003, CN: "api.example.com", Validation: "dv", Deployed: []report.Cert{cert("api.example.com", "cc", "RSA 2048", 3*day)}},
|
||||
{ID: 1003, CN: "api.example.com", Validation: "dv", Deployed: []report.Cert{cert("api.example.com", "cc", "RSA 2048", day)}},
|
||||
{ID: 1004, CN: "new.example.com", Validation: "dv", Pending: 1, Deployed: []report.Cert{}},
|
||||
}})
|
||||
got := srv.akamaiConditions(now)
|
||||
@@ -100,7 +100,7 @@ func TestAkamaiCPS(t *testing.T) {
|
||||
t.Fatalf("%s", k)
|
||||
}
|
||||
if !strings.Contains(got[0].Detail, "third-party certificate") || !strings.Contains(got[1].Detail, "signed by your certificate authority") ||
|
||||
!strings.Contains(got[2].Detail, "stalled") || !strings.HasPrefix(got[2].Title, "Akamai certificate for api.example.com: ends in 3 days") {
|
||||
!strings.Contains(got[2].Detail, "stalled") || !strings.HasPrefix(got[2].Title, "Akamai certificate for api.example.com: ends in ") || !strings.Contains(got[2].Title, " hours") {
|
||||
t.Errorf("%+v", got)
|
||||
}
|
||||
rows := srv.certificates(now)
|
||||
|
||||
@@ -217,6 +217,9 @@ type Condition struct {
|
||||
Affected string `json:"affected,omitempty"` // what else this takes down (root-cause grouping)
|
||||
Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it, when any did
|
||||
Map string `json:"map,omitempty"` // where it is on the maps: a page path with its focus
|
||||
// Ends is when the certificate a condition is about ends, for one that
|
||||
// is: certSeverity keeps it a warning until it's close (#34).
|
||||
Ends time.Time `json:"-"`
|
||||
}
|
||||
|
||||
// view judges every host now: what the UI shows, and the conditions the
|
||||
@@ -364,6 +367,7 @@ func (srv *Server) view(now time.Time) ([]HostView, []Condition) {
|
||||
conds = append(conds, dlConds...)
|
||||
conds = append(conds, srv.workloadConditions(now)...)
|
||||
conds = append(conds, srv.certConditions(now)...)
|
||||
certSeverity(conds, now, srv.certCritical())
|
||||
if byHost, _ := srv.vmLinks(); len(byHost) > 0 {
|
||||
explainSilent(conds, byHost)
|
||||
for i := range hosts {
|
||||
|
||||
+49
-13
@@ -22,11 +22,43 @@ import (
|
||||
// with each source (decision 5); this page only gathers.
|
||||
|
||||
const (
|
||||
certWarn = 21 * 24 * time.Hour
|
||||
certCritical = 7 * 24 * time.Hour
|
||||
certPlaces = 50 // places sent per certificate; PlaceCount has them all
|
||||
certWarn = 21 * 24 * time.Hour
|
||||
certPlaces = 50 // places sent per certificate; PlaceCount has them all
|
||||
// defaultCertCritical is certificates.critical_within's default: four
|
||||
// days left is a job for working hours, two is not (#34).
|
||||
defaultCertCritical = 48 * time.Hour
|
||||
)
|
||||
|
||||
// CertificatesConfig is obs-beacond.yaml's certificates section.
|
||||
type CertificatesConfig struct {
|
||||
// CriticalWithin is how close to its end a certificate becomes
|
||||
// critical (48h). Before that it's a warning, from whenever its source
|
||||
// starts warning (21 days for most).
|
||||
CriticalWithin time.Duration `yaml:"critical_within"`
|
||||
}
|
||||
|
||||
func (srv *Server) certCritical() time.Duration {
|
||||
if srv.cfg == nil || srv.cfg.Certificates.CriticalWithin <= 0 {
|
||||
return defaultCertCritical
|
||||
}
|
||||
return srv.cfg.Certificates.CriticalWithin
|
||||
}
|
||||
|
||||
// certSeverity holds every certificate condition at warning until its
|
||||
// certificate is within critical of its end, whatever its source judged
|
||||
// (each has its own critical threshold: 7 days for checks and Vault, 3
|
||||
// for Cloudflare). A source's critical inside the window stands, and so
|
||||
// does one about something other than the end (an ended certificate is
|
||||
// past it).
|
||||
func certSeverity(conds []Condition, now time.Time, critical time.Duration) {
|
||||
for i := range conds {
|
||||
c := &conds[i]
|
||||
if c.Severity == "critical" && !c.Ends.IsZero() && c.Ends.Sub(now) > critical {
|
||||
c.Severity = "warning"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CertInvPlace is where a certificate was seen.
|
||||
type CertInvPlace struct {
|
||||
Kind string `json:"kind"` // "host", "file", "store", "vault", "check", "cloudflare", "cluster", "akamai"
|
||||
@@ -88,11 +120,12 @@ type CertReload struct {
|
||||
// reload before it's said not to have happened.
|
||||
const reloadGrace = time.Hour
|
||||
|
||||
// certStatus judges a certificate by the time it has left: 21 and 7 days,
|
||||
// or a third and a tenth of a short life (the same as Vault's leaves).
|
||||
func certStatus(c report.Cert, now time.Time) (string, string) {
|
||||
// certStatus judges a certificate by the time it has left: 21 days, and
|
||||
// critical (certificates.critical_within), or a third and a tenth of a
|
||||
// short life (the same as Vault's leaves).
|
||||
func certStatus(c report.Cert, now time.Time, critical time.Duration) (string, string) {
|
||||
left, life := c.NotAfter.Sub(now), c.NotAfter.Sub(c.NotBefore)
|
||||
warn, crit := certWarn, certCritical
|
||||
warn, crit := certWarn, critical
|
||||
if life > 0 && !c.CA {
|
||||
warn, crit = min(warn, life/3), min(crit, life/10)
|
||||
}
|
||||
@@ -296,9 +329,9 @@ func (srv *Server) certificates(now time.Time) []*CertInv {
|
||||
if row.NotAfter.Before(now) {
|
||||
row.Days = -int(now.Sub(row.NotAfter).Hours()/24) - 1
|
||||
}
|
||||
row.Status, row.Word = certStatus(row.Cert, now)
|
||||
if fileOnly(row) && row.Status == "warning" {
|
||||
row.Status = "good" // a file no port serves is judged at 7 days only
|
||||
row.Status, row.Word = certStatus(row.Cert, now, srv.certCritical())
|
||||
if fileOnly(row) && row.Status == "warning" && row.NotAfter.Sub(now) >= fileWarn {
|
||||
row.Status = "good" // a file no port serves is judged from 7 days only
|
||||
}
|
||||
if row.Weak = certinfo.Weak(&row.Cert); row.Weak != "" && row.Status == "good" && !fileOnly(row) {
|
||||
row.Status = "warning"
|
||||
@@ -373,9 +406,12 @@ func namesKey(c report.Cert) string {
|
||||
return strings.Join(c.Names, ",")
|
||||
}
|
||||
|
||||
// fileWarn is when a certificate found only in files starts to warn: 7
|
||||
// days, not 21 (one no port serves may be nobody's).
|
||||
const fileWarn = 7 * 24 * time.Hour
|
||||
|
||||
// fileOnly says a certificate was found only in files or a Windows
|
||||
// certificate store: judged at 7 days, not 21 (one no port serves may be
|
||||
// nobody's).
|
||||
// certificate store.
|
||||
func fileOnly(c *CertInv) bool {
|
||||
for _, k := range c.Kinds {
|
||||
if k != "file" && k != "store" {
|
||||
@@ -528,7 +564,7 @@ func (srv *Server) certConditions(now time.Time) []Condition {
|
||||
if !own {
|
||||
sev = "warning" // the others alert on its end; this is only the reload
|
||||
}
|
||||
cond := Condition{Key: "certinv/" + c.Fingerprint, Severity: sev, Title: "Certificate " + c.Subject + " " + what,
|
||||
cond := Condition{Key: "certinv/" + c.Fingerprint, Severity: sev, Ends: c.NotAfter, Title: "Certificate " + c.Subject + " " + what,
|
||||
Detail: strings.Join(detail, " "), Href: "/certificates?q=" + c.Fingerprint[:16]}
|
||||
if len(hosts) == 1 {
|
||||
for h := range hosts {
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestCertificates(t *testing.T) {
|
||||
srv.store.Put("node1", results(now.Add(time.Second)), now.Add(time.Second)) // kept between scans
|
||||
|
||||
conds := srv.certConditions(now)
|
||||
if len(conds) != 1 || conds[0].Key != "certinv/"+mine.Fingerprint || conds[0].Severity != "critical" || conds[0].Host != "node1" ||
|
||||
if len(conds) != 1 || conds[0].Key != "certinv/"+mine.Fingerprint || conds[0].Severity != "warning" || conds[0].Host != "node1" || // 4 days: a warning (#34)
|
||||
conds[0].Detail != "Served on node1:8443 · admin, node1:25 (smtp STARTTLS) · postfix." {
|
||||
t.Errorf("conditions: %+v", conds)
|
||||
}
|
||||
@@ -136,7 +136,7 @@ func TestCertFiles(t *testing.T) {
|
||||
for _, c := range srv.certificates(now) {
|
||||
rows[c.Fingerprint] = c
|
||||
}
|
||||
if c := rows[old.Fingerprint]; c == nil || c.Status != "critical" || len(c.NotReloaded) != 1 || c.NotReloaded[0] != "node1:8443 · admin" ||
|
||||
if c := rows[old.Fingerprint]; c == nil || c.Status != "warning" || len(c.NotReloaded) != 1 || c.NotReloaded[0] != "node1:8443 · admin" ||
|
||||
!strings.HasPrefix(c.Renewed, "node1:/etc/letsencrypt/live/admin/cert.pem (ends ") {
|
||||
t.Errorf("the admin's: %+v", c)
|
||||
}
|
||||
@@ -154,14 +154,14 @@ func TestCertFiles(t *testing.T) {
|
||||
if len(conds) != 3 {
|
||||
t.Errorf("%d conditions: %+v", len(conds), conds)
|
||||
}
|
||||
if c := conds["certinv/"+old.Fingerprint]; c.Severity != "critical" || c.Host != "node1" || !strings.HasSuffix(c.Title, "; renewed but not reloaded") ||
|
||||
if c := conds["certinv/"+old.Fingerprint]; c.Severity != "warning" || c.Host != "node1" || !strings.HasSuffix(c.Title, "; renewed but not reloaded") ||
|
||||
!strings.Contains(c.Detail, "but node1:8443 · admin still serves this one: reload the service.") {
|
||||
t.Errorf("the admin's: %+v", c)
|
||||
}
|
||||
if c := conds["certinv/"+shop.Fingerprint]; c.Severity != "warning" || c.Title != "Certificate shop.example.org renewed but not reloaded" {
|
||||
t.Errorf("the shop's, which a check alerts on: %+v", c)
|
||||
}
|
||||
if c := conds["certinv/"+ending.Fingerprint]; c.Severity != "critical" || c.Detail != "In node1:/etc/pki/tls/certs/ending.crt; no port serves it." {
|
||||
if c := conds["certinv/"+ending.Fingerprint]; c.Severity != "warning" || c.Detail != "In node1:/etc/pki/tls/certs/ending.crt; no port serves it." {
|
||||
t.Errorf("a file ending in 3 days: %+v", c)
|
||||
}
|
||||
|
||||
@@ -264,3 +264,29 @@ func TestCertWrong(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A certificate's alert is a warning until it's within critical_within of
|
||||
// its end, whichever source judged it critical (#34).
|
||||
func TestCertSeverity(t *testing.T) {
|
||||
now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC)
|
||||
conds := []Condition{
|
||||
{Key: "four days", Severity: "critical", Ends: now.Add(4 * 24 * time.Hour)},
|
||||
{Key: "a day", Severity: "critical", Ends: now.Add(24 * time.Hour)},
|
||||
{Key: "ended", Severity: "critical", Ends: now.Add(-time.Hour)},
|
||||
{Key: "not a certificate", Severity: "critical"},
|
||||
{Key: "warning", Severity: "warning", Ends: now.Add(10 * 24 * time.Hour)},
|
||||
}
|
||||
certSeverity(conds, now, defaultCertCritical)
|
||||
want := []string{"warning", "critical", "critical", "critical", "warning"}
|
||||
for i, c := range conds {
|
||||
if c.Severity != want[i] {
|
||||
t.Errorf("%s: %s, want %s", c.Key, c.Severity, want[i])
|
||||
}
|
||||
}
|
||||
if st, _ := certStatus(report.Cert{NotBefore: now.Add(-300 * 24 * time.Hour), NotAfter: now.Add(4 * 24 * time.Hour)}, now, defaultCertCritical); st != "warning" {
|
||||
t.Errorf("certStatus four days out: %s", st)
|
||||
}
|
||||
if st, _ := certStatus(report.Cert{NotBefore: now.Add(-300 * 24 * time.Hour), NotAfter: now.Add(4 * 24 * time.Hour)}, now, 7*24*time.Hour); st != "critical" {
|
||||
t.Errorf("certStatus four days out, critical_within 7d: %s", st)
|
||||
}
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func (srv *Server) judgeChecks(now time.Time) ([]CheckView, []Condition) {
|
||||
sev = "warning"
|
||||
}
|
||||
if sev != "" {
|
||||
conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev,
|
||||
conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, Ends: soonest,
|
||||
Title: fmt.Sprintf("%s certificate expires in %s", s.Name, plural(days, "day", "days")),
|
||||
Detail: fmt.Sprintf("Seen on %s; it expires %s.", s.Target(), soonest.UTC().Format("2006-01-02 15:04 UTC")), Href: href})
|
||||
if v.Status == "good" {
|
||||
|
||||
@@ -108,11 +108,11 @@ func (srv *Server) cloudflareConditions(now time.Time) []Condition {
|
||||
Detail: "Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
|
||||
case c.Expires.IsZero():
|
||||
case days < cfg.CertCriticalDays:
|
||||
out = append(out, Condition{Key: key, Severity: "critical",
|
||||
out = append(out, Condition{Key: key, Severity: "critical", Ends: c.Expires,
|
||||
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
|
||||
Detail: "Cloudflare normally renews well before this; renewal is stuck. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
|
||||
case days < cfg.CertWarnDays:
|
||||
out = append(out, Condition{Key: key, Severity: "warning",
|
||||
out = append(out, Condition{Key: key, Severity: "warning", Ends: c.Expires,
|
||||
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
|
||||
Detail: "Cloudflare normally renews 30 days ahead. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
|
||||
}
|
||||
|
||||
@@ -74,6 +74,8 @@ type Config struct {
|
||||
Mail *mailmon.Config `yaml:"mail"`
|
||||
// Reports' timezone and the weekly digest.
|
||||
Reports *ReportsConfig `yaml:"reports"`
|
||||
// How close to its end a certificate becomes critical, from any source.
|
||||
Certificates CertificatesConfig `yaml:"certificates"`
|
||||
// Vault and OpenBao clusters obs-beacond reads itself (docs/vault.md).
|
||||
Vault []vault.Config `yaml:"vault"`
|
||||
}
|
||||
@@ -142,6 +144,12 @@ func LoadConfig(p string) (*Config, error) {
|
||||
if c.StaleAfter == 0 {
|
||||
c.StaleAfter = 60 * time.Second
|
||||
}
|
||||
if c.Certificates.CriticalWithin < 0 {
|
||||
return nil, fmt.Errorf("%s: certificates.critical_within can't be negative", p)
|
||||
}
|
||||
if c.Certificates.CriticalWithin == 0 {
|
||||
c.Certificates.CriticalWithin = defaultCertCritical
|
||||
}
|
||||
if c.EmergencyListen != "" && !loopback(c.EmergencyListen) {
|
||||
return nil, fmt.Errorf("%s: emergency_listen must be a loopback address (it's reached through an SSH tunnel), not %s", p, c.EmergencyListen)
|
||||
}
|
||||
|
||||
@@ -855,7 +855,8 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
|
||||
if on, _ := r.Facts["served_on"].(string); on != "" {
|
||||
detail = append(detail, "Served on "+on+".")
|
||||
}
|
||||
add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "),
|
||||
ends, _ := time.Parse(time.RFC3339, fmt.Sprint(r.Facts["ends"]))
|
||||
add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends,
|
||||
Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind)
|
||||
}
|
||||
case report.KindVM:
|
||||
|
||||
Reference in new issue
Block a user