Demo: problems come round about on a schedule, not like a clock #64
No files matched your search
+7
-3
@@ -61,9 +61,13 @@ Most of the simulated fleet is healthy. What's wrong follows a plan
|
||||
(`scripts/fleet/scenario.go`, and "The demo fleet" in development.md):
|
||||
three stories with a cause and its effects, a few lasting warnings, and
|
||||
problems that come and go through the day, so incidents open, resolve and
|
||||
show up under Recently resolved. Because the plan runs on the clock, two
|
||||
visitors an hour apart see different problems, and a restarted demo shows
|
||||
the same ones the old one would have.
|
||||
show up under Recently resolved. Each comes round about on a schedule but
|
||||
not like a clock: its start and length vary, and it skips about half its
|
||||
turns, four in five at night in Phoenix, so the history has a working
|
||||
day's shape, about 1,400 incidents a month rather than thousands, and in
|
||||
the day there's never an hour with nothing going on. Because the plan
|
||||
runs on the clock, two visitors an hour apart see different problems, and
|
||||
a restarted demo shows the same ones the old one would have.
|
||||
|
||||
The demo starts with a month behind it, worked out from the same plan and
|
||||
clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and
|
||||
|
||||
@@ -120,8 +120,8 @@ func (c *fakeCluster) play(now time.Time) {
|
||||
w := cloneWorkload(e.good)
|
||||
if on {
|
||||
e.bad(&w, e.s.began(now))
|
||||
} else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every {
|
||||
w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended
|
||||
} else if o := e.s.last(now); !e.good.Since.IsZero() && now.Sub(o.start) < e.s.every {
|
||||
w.Since = o.end // right again since the spell ended
|
||||
}
|
||||
c.recs[e.id] = w
|
||||
c.changed[e.id] = true
|
||||
|
||||
@@ -361,11 +361,11 @@ func (d *demoHost) containers(r *report.Report, now time.Time) {
|
||||
if c.Name != "app-04" {
|
||||
continue
|
||||
}
|
||||
if b := spellAppExits.began(now); spellAppExits.active(now) {
|
||||
c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), ""
|
||||
if o := spellAppExits.last(now); spellAppExits.active(now) {
|
||||
c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(o.start))), ""
|
||||
} else {
|
||||
c.State, c.Health = "running", "healthy"
|
||||
c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on))))
|
||||
c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(o.end)))
|
||||
}
|
||||
}
|
||||
r.Containers = &report.Containers{List: list}
|
||||
@@ -392,11 +392,11 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
|
||||
for _, u := range demoUnits(d.n) {
|
||||
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()}
|
||||
if d.n == demoFailedUnit && u == "redis-server" {
|
||||
switch b := spellRedisFails.began(now); {
|
||||
switch o := spellRedisFails.last(now); {
|
||||
case spellRedisFails.active(now):
|
||||
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
|
||||
case b.After(start): // restarted by hand when the spell ended
|
||||
un.Since = b.Add(spellRedisFails.on).UTC()
|
||||
case o.start.After(start): // restarted by hand when the spell ended
|
||||
un.Since = o.end.UTC()
|
||||
}
|
||||
}
|
||||
r.Units = append(r.Units, un)
|
||||
|
||||
+16
-11
@@ -89,9 +89,11 @@ func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) er
|
||||
state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"})
|
||||
for _, fl := range pastFlaps {
|
||||
s := fl.s
|
||||
for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) {
|
||||
statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status},
|
||||
state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after})
|
||||
for _, o := range s.between(from, now) {
|
||||
if o.end.Before(now) {
|
||||
statuses = append(statuses, state.Change{Entity: fl.entity, At: o.start.Add(20 * time.Second), Status: fl.status},
|
||||
state.Change{Entity: fl.entity, At: o.end.Add(20 * time.Second), Status: fl.after})
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) })
|
||||
@@ -245,9 +247,9 @@ func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour
|
||||
// obs-beacond where the numbers carry on.
|
||||
func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
|
||||
type occ struct {
|
||||
p past
|
||||
began time.Time
|
||||
first bool // the first of its kind on its day: it gets the note
|
||||
p past
|
||||
began, end time.Time
|
||||
first bool // the first of its kind on its day: it gets the note
|
||||
}
|
||||
var all []occ
|
||||
for _, p := range pastProblems {
|
||||
@@ -255,9 +257,12 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
|
||||
continue
|
||||
}
|
||||
day := ""
|
||||
for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) {
|
||||
d := b.UTC().Format("2006-01-02")
|
||||
all = append(all, occ{p, b, d != day})
|
||||
for _, o := range p.s.between(from, now) {
|
||||
if !o.end.Before(now) {
|
||||
continue // still going: the live demo opens it
|
||||
}
|
||||
d := o.start.UTC().Format("2006-01-02")
|
||||
all = append(all, occ{p, o.start, o.end, d != day})
|
||||
day = d
|
||||
}
|
||||
}
|
||||
@@ -268,10 +273,10 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
|
||||
// resolved within a minute of its end.
|
||||
opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second)
|
||||
fired := opened.Add(2 * time.Minute)
|
||||
resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second)
|
||||
resolved := o.end.Add(time.Duration(15+seedInt(p.key+" end", o.began, 45)) * time.Second)
|
||||
inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail,
|
||||
Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved}
|
||||
if p.sev == "critical" && p.s.on > 10*time.Minute {
|
||||
if p.sev == "critical" && o.end.Sub(o.began) > 10*time.Minute {
|
||||
inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute)
|
||||
}
|
||||
if err := db.PutIncident(inc); err != nil {
|
||||
|
||||
+77
-16
@@ -1,6 +1,10 @@
|
||||
package main
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What goes wrong in the demo, and when (#35). A real fleet is mostly
|
||||
// fine: a few things stay wrong until someone gets to them, and the rest
|
||||
@@ -9,10 +13,10 @@ import "time"
|
||||
// - three stories, each a cause with its effects:
|
||||
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
|
||||
// slow while the drive's data is rebuilt (all the time);
|
||||
// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica,
|
||||
// 2. checkout errors: about every 2 hours phx1-db03, a PostgreSQL replica,
|
||||
// falls behind, HAProxy takes it out, and the shop's checkout answers
|
||||
// with errors until it catches up (20 minutes);
|
||||
// 3. the legacy datacenter: every 4 hours its server stops answering
|
||||
// 3. the legacy datacenter: about every 4 hours its server stops answering
|
||||
// Akamai's liveness test, GTM has nowhere to send the legacy property,
|
||||
// and its CP code fails (30 minutes);
|
||||
// - a few lasting warnings anyone would have: a failover nobody moved
|
||||
@@ -27,8 +31,16 @@ import "time"
|
||||
// picks up where the clock is, and the history the demo starts with can
|
||||
// be worked out for any time in the past.
|
||||
|
||||
// spell is when one problem is happening: for on, once every every, at
|
||||
// into each period. A zero every means all the time.
|
||||
// spell is when one problem happens: about once every every, about at
|
||||
// into each period, for about on. A zero every means all the time.
|
||||
//
|
||||
// Not like a clock (#57): each period's start moves by up to an eighth of
|
||||
// the period, its length is 70% to 130% of on, and half the periods have
|
||||
// none, four in five at night in Phoenix, where the fleet's main site is.
|
||||
// That still starts something every 20 minutes or so, while 30 days of
|
||||
// history hold about 1,400 incidents rather than 3,700. All of it is
|
||||
// seeded by the spell and the period, so it's the same whenever it's
|
||||
// worked out.
|
||||
type spell struct {
|
||||
every, at, on time.Duration
|
||||
}
|
||||
@@ -36,26 +48,75 @@ type spell struct {
|
||||
// always is a problem that's there all the time.
|
||||
var always = spell{}
|
||||
|
||||
// occurrence is one spell, from start to end.
|
||||
type occurrence struct{ start, end time.Time }
|
||||
|
||||
var phoenix = time.FixedZone("MST", -7*3600)
|
||||
|
||||
// nth is the spell in period n, counting from the Unix epoch; false when
|
||||
// that period has none.
|
||||
func (s spell) nth(n int64) (occurrence, bool) {
|
||||
h := fnv.New64a()
|
||||
fmt.Fprintf(h, "%d %d %d %d", s.every, s.at, s.on, n)
|
||||
x := h.Sum64() // FNV leaves short, alike inputs' high bits alike: mix them
|
||||
x ^= x >> 30
|
||||
x *= 0xbf58476d1ce4e5b9
|
||||
x ^= x >> 27
|
||||
x *= 0x94d049bb133111eb
|
||||
x ^= x >> 31
|
||||
frac := func(shift uint) float64 { return float64((x>>shift)&0xffff) / 0x10000 }
|
||||
on := time.Duration(float64(s.on) * (0.7 + 0.6*frac(0)))
|
||||
at := s.at + time.Duration((2*frac(16)-1)*float64(s.every/8))
|
||||
at = max(0, min(at, s.every-on)) // inside its own period
|
||||
start := time.Unix(0, n*int64(s.every)).Add(at)
|
||||
skip := 0.5
|
||||
if hr := start.In(phoenix).Hour(); hr < 7 || hr >= 22 {
|
||||
skip = 0.8
|
||||
}
|
||||
if frac(32) < skip {
|
||||
return occurrence{}, false
|
||||
}
|
||||
return occurrence{start, start.Add(on)}, true
|
||||
}
|
||||
|
||||
func (s spell) period(t time.Time) int64 { return t.UnixNano() / int64(s.every) }
|
||||
|
||||
func (s spell) active(t time.Time) bool {
|
||||
if s.every == 0 {
|
||||
return true
|
||||
}
|
||||
p := time.Duration(t.UnixNano()) % s.every
|
||||
return p >= s.at && p < s.at+s.on
|
||||
o, ok := s.nth(s.period(t))
|
||||
return ok && !t.Before(o.start) && t.Before(o.end)
|
||||
}
|
||||
|
||||
// began is when the current spell started, or the last one if it's over;
|
||||
// a long time ago for a problem that's always there.
|
||||
func (s spell) began(t time.Time) time.Time {
|
||||
// last is the spell going on at t, or the last one before it; for a
|
||||
// problem that's always there, one that began long ago and hasn't ended.
|
||||
func (s spell) last(t time.Time) occurrence {
|
||||
if s.every == 0 {
|
||||
return t.Add(-9 * 24 * time.Hour)
|
||||
return occurrence{t.Add(-9 * 24 * time.Hour), t.Add(100 * 365 * 24 * time.Hour)}
|
||||
}
|
||||
p := time.Duration(t.UnixNano()) % s.every
|
||||
start := t.Add(-p + s.at)
|
||||
if start.After(t) {
|
||||
start = start.Add(-s.every)
|
||||
for n := s.period(t); ; n-- {
|
||||
if o, ok := s.nth(n); ok && !o.start.After(t) {
|
||||
return o
|
||||
}
|
||||
}
|
||||
return start
|
||||
}
|
||||
|
||||
// began is when the current spell started, or the last one if it's over.
|
||||
func (s spell) began(t time.Time) time.Time { return s.last(t).start }
|
||||
|
||||
// between is every spell that starts from from to to, oldest first.
|
||||
func (s spell) between(from, to time.Time) []occurrence {
|
||||
if s.every == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []occurrence
|
||||
for n := s.period(from); n <= s.period(to); n++ {
|
||||
if o, ok := s.nth(n); ok && !o.start.Before(from) && !o.start.After(to) {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The spells, by story. Periods aren't multiples of each other, so the
|
||||
|
||||
@@ -7,33 +7,67 @@ import (
|
||||
"github.com/cairnobs/cairnobs/internal/report"
|
||||
)
|
||||
|
||||
// A spell comes round about every period, never like a clock (#57), and
|
||||
// active, last and between always agree.
|
||||
func TestSpell(t *testing.T) {
|
||||
s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute}
|
||||
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
at time.Duration
|
||||
active bool
|
||||
began time.Duration
|
||||
}{
|
||||
{19 * time.Minute, false, -40 * time.Minute},
|
||||
{20 * time.Minute, true, 20 * time.Minute},
|
||||
{29 * time.Minute, true, 20 * time.Minute},
|
||||
{30 * time.Minute, false, 20 * time.Minute},
|
||||
{80 * time.Minute, true, 80 * time.Minute},
|
||||
} {
|
||||
now := day.Add(c.at)
|
||||
if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) {
|
||||
t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day))
|
||||
day := time.Date(2026, 11, 16, 7, 0, 0, 0, time.UTC) // midnight in Phoenix
|
||||
week := s.between(day, day.Add(7*24*time.Hour))
|
||||
starts := map[time.Duration]bool{}
|
||||
night, nights, daytime, days := 0, 0, 0, 0
|
||||
for i, o := range week {
|
||||
if l := o.end.Sub(o.start); l < 7*time.Minute || l > 13*time.Minute {
|
||||
t.Errorf("%s lasts %s", o.start, l)
|
||||
}
|
||||
if s.period(o.start) != s.period(o.end.Add(-time.Nanosecond)) {
|
||||
t.Errorf("%s runs into the next period", o.start)
|
||||
}
|
||||
if i > 0 && !week[i-1].end.Before(o.start) {
|
||||
t.Errorf("%s overlaps the one before", o.start)
|
||||
}
|
||||
starts[time.Duration(o.start.UnixNano())%s.every/time.Minute] = true
|
||||
mid := o.start.Add(o.end.Sub(o.start) / 2)
|
||||
if !s.active(mid) || !s.last(mid).start.Equal(o.start) || !s.began(o.end).Equal(o.start) || s.active(o.end) {
|
||||
t.Errorf("%s: active %v last %s", mid, s.active(mid), s.last(mid).start)
|
||||
}
|
||||
if hr := o.start.In(phoenix).Hour(); hr < 7 || hr >= 22 {
|
||||
night++
|
||||
} else {
|
||||
daytime++
|
||||
}
|
||||
}
|
||||
if !always.active(day) || !always.began(day).Before(day) {
|
||||
for h := 0; h < 7*24; h++ {
|
||||
if hr := day.Add(time.Duration(h) * time.Hour).In(phoenix).Hour(); hr < 7 || hr >= 22 {
|
||||
nights++
|
||||
} else {
|
||||
days++
|
||||
}
|
||||
}
|
||||
if n := len(week); n < 7*24/5 || n > 7*24*2/3 {
|
||||
t.Errorf("%d of %d hours had it", n, 7*24)
|
||||
}
|
||||
if float64(night)/float64(nights) >= float64(daytime)/float64(days) {
|
||||
t.Errorf("as often at night (%d/%d) as by day (%d/%d)", night, nights, daytime, days)
|
||||
}
|
||||
if len(starts) < 5 {
|
||||
t.Errorf("starts at only %d minutes of the hour", len(starts))
|
||||
}
|
||||
// Between spells, the last one is the one before.
|
||||
for i := 1; i < len(week); i++ {
|
||||
gap := week[i-1].end.Add(time.Second)
|
||||
if gap.Before(week[i].start) && (s.active(gap) || !s.last(gap).start.Equal(week[i-1].start)) {
|
||||
t.Errorf("%s: active %v last %s", gap, s.active(gap), s.last(gap).start)
|
||||
}
|
||||
}
|
||||
if !always.active(day) || !always.began(day).Before(day) || always.between(day, day.Add(time.Hour)) != nil {
|
||||
t.Error("always")
|
||||
}
|
||||
}
|
||||
|
||||
// The demo stays calm: across a day, no more than a few of the problems
|
||||
// that come and go are on at once, and every one of them is on some of
|
||||
// the time and off the rest.
|
||||
// The demo stays calm: across three days, no more than a few of the
|
||||
// problems that come and go are on at once, every one of them is on some
|
||||
// of the time and off the rest, and in the Phoenix day a visitor never
|
||||
// goes an hour without one.
|
||||
func TestScenarioIsCalm(t *testing.T) {
|
||||
spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits,
|
||||
"502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory,
|
||||
@@ -43,7 +77,9 @@ func TestScenarioIsCalm(t *testing.T) {
|
||||
on := map[string]int{}
|
||||
most, total, steps := 0, 0, 0
|
||||
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
|
||||
for m := 0; m < 24*60; m++ {
|
||||
const minutes = 3 * 24 * 60
|
||||
quiet, longest := 0, 0
|
||||
for m := 0; m < minutes; m++ {
|
||||
now := day.Add(time.Duration(m) * time.Minute)
|
||||
n := 0
|
||||
for name, s := range spells {
|
||||
@@ -56,12 +92,22 @@ func TestScenarioIsCalm(t *testing.T) {
|
||||
}
|
||||
}
|
||||
most, total, steps = max(most, n), total+n, steps+1
|
||||
if hr := now.In(phoenix).Hour(); n == 0 && hr >= 7 && hr < 22 {
|
||||
quiet++
|
||||
longest = max(longest, quiet)
|
||||
} else {
|
||||
quiet = 0
|
||||
}
|
||||
}
|
||||
for name := range spells {
|
||||
if on[name] == 0 || on[name] == 24*60 {
|
||||
t.Errorf("%s is on %d minutes of the day", name, on[name])
|
||||
if on[name] == 0 || on[name] == minutes {
|
||||
t.Errorf("%s is on %d minutes of three days", name, on[name])
|
||||
}
|
||||
}
|
||||
t.Logf("longest quiet daytime stretch %d min; %.1f on at once on average, %d at most", longest, float64(total)/float64(steps), most)
|
||||
if longest >= 60 {
|
||||
t.Errorf("%d minutes in the day with nothing going on", longest)
|
||||
}
|
||||
if avg := float64(total) / float64(steps); avg > 4 || most > 9 {
|
||||
t.Errorf("on at once: %.1f on average, %d at most", avg, most)
|
||||
}
|
||||
@@ -82,13 +128,13 @@ func TestFlapSet(t *testing.T) {
|
||||
if len(c.episodes) != 2 {
|
||||
t.Fatalf("%d episodes, want a and q", len(c.episodes))
|
||||
}
|
||||
hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC)
|
||||
c.play(hour.Add(5 * time.Minute))
|
||||
o := s.between(time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC), time.Date(2026, 11, 17, 3, 0, 0, 0, time.UTC))[0]
|
||||
c.play(o.start.Add(time.Minute))
|
||||
if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" {
|
||||
t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"])
|
||||
}
|
||||
c.removed = nil
|
||||
c.play(hour.Add(40 * time.Minute))
|
||||
c.play(o.end.Add(time.Minute))
|
||||
if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 {
|
||||
t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed)
|
||||
}
|
||||
|
||||
@@ -74,13 +74,16 @@ func newWindowsHost(name string, interval time.Duration) *host {
|
||||
// them.
|
||||
func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
|
||||
w := wh.w
|
||||
crashes, lastCrash := 0, time.Time{}
|
||||
crashes, lastCrash, firstCrash := 0, time.Time{}, time.Time{}
|
||||
if w.name == "win-fs01" {
|
||||
// It goes down without shutting down cleanly and is straight back:
|
||||
// up since the last time.
|
||||
lastCrash = spellFSCrash.began(now)
|
||||
for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) {
|
||||
crashes++
|
||||
day := spellFSCrash.between(now.Add(-24*time.Hour+time.Nanosecond), now)
|
||||
crashes = max(1, len(day))
|
||||
firstCrash = lastCrash
|
||||
if len(day) > 0 {
|
||||
firstCrash = day[0].start
|
||||
}
|
||||
h.start = lastCrash
|
||||
r.Uptime = max(0, now.Sub(h.start).Seconds())
|
||||
@@ -89,10 +92,10 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
|
||||
for _, u := range w.units {
|
||||
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)}
|
||||
if w.name == "win-sql01" && u == "SQLSERVERAGENT" {
|
||||
if b := spellSQLAgent.began(now); spellSQLAgent.active(now) {
|
||||
if o := spellSQLAgent.last(now); spellSQLAgent.active(now) {
|
||||
un.Active, un.Sub, un.Since = "failed", "dead", time.Time{}
|
||||
} else if b.After(h.start) { // someone started it again
|
||||
un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second)
|
||||
} else if o.start.After(h.start) { // someone started it again
|
||||
un.Since = o.end.UTC().Truncate(time.Second)
|
||||
}
|
||||
}
|
||||
r.Units = append(r.Units, un)
|
||||
@@ -104,7 +107,7 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
|
||||
}
|
||||
switch w.name {
|
||||
case "win-fs01":
|
||||
first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every)
|
||||
first := firstCrash
|
||||
group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start),
|
||||
"The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.")
|
||||
group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`)
|
||||
|
||||
Reference in new issue
Block a user