// obs-beacon-agent is the one agent Cairn OBS runs on each host. It reads the host // and its containers and pushes a report to obs-beacond every interval. // // obs-beacon-agent -config /etc/obs-beacon/agent.yaml collect and report (unprivileged) // obs-beacon-agent helper -listen ... -docker ... the read-only root helper // obs-beacon-agent scan -config ... -host ... -out ... one backup scan (root, on a timer) // obs-beacon-agent join [-force] join a server with a token from Hosts, Add host (root) // obs-beacon-agent service install|uninstall Windows: install or remove the service (Administrator) // obs-beacon-agent cluster the cluster agent, in a cluster (deploy/kubernetes/cluster-agent) package main import ( "bytes" "context" "encoding/json" "errors" "flag" "fmt" "io" "log" "math/rand/v2" "net/http" "os" "os/exec" "os/signal" "path/filepath" "runtime" "strings" "syscall" "time" "gopkg.in/yaml.v3" "github.com/cairnobs/cairnobs/internal/agentlink" "github.com/cairnobs/cairnobs/internal/certfiles" "github.com/cairnobs/cairnobs/internal/certreload" "github.com/cairnobs/cairnobs/internal/check" "github.com/cairnobs/cairnobs/internal/collect" "github.com/cairnobs/cairnobs/internal/docker" "github.com/cairnobs/cairnobs/internal/fetch" "github.com/cairnobs/cairnobs/internal/hmc" "github.com/cairnobs/cairnobs/internal/ibmi" "github.com/cairnobs/cairnobs/internal/libvirt" "github.com/cairnobs/cairnobs/internal/netmap" "github.com/cairnobs/cairnobs/internal/nutanix" "github.com/cairnobs/cairnobs/internal/proxmox" "github.com/cairnobs/cairnobs/internal/purerest" "github.com/cairnobs/cairnobs/internal/report" "github.com/cairnobs/cairnobs/internal/tlsscan" "github.com/cairnobs/cairnobs/internal/units" "github.com/cairnobs/cairnobs/internal/version" "github.com/cairnobs/cairnobs/internal/vsphere" "github.com/cairnobs/cairnobs/internal/watch" "github.com/cairnobs/cairnobs/internal/winevent" ) type config struct { Server string `yaml:"server"` // obs-beacond's agent listener, e.g. https://10.0.0.3:8461 CAFingerprint string `yaml:"ca_fingerprint"` // the agent CA, pinned (`obs-beacond ca` prints it); needed with https TokenFile string `yaml:"token_file"` // this agent's token: enrolls over https, signs every request over http StateDir string `yaml:"state_dir"` // the agent's certificate and key; default /var/lib/obs-beacon-agent Host string `yaml:"host"` // defaults to the hostname Interval time.Duration `yaml:"interval"` // defaults to 15s HelperSocket string `yaml:"helper_socket"` // empty: no containers and no map sample BackupScan string `yaml:"backup_scan"` // the backup scan's output; empty: none // Map: false stops the map sample (docs/maps.md) on this host: no socket // sample and no WireGuard read. Default true when there's a helper. Map *bool `yaml:"map"` // CertScan: false stops trying a TLS handshake on each listening port // once a day for the certificate inventory (docs/cert-inventory.md). // Default true when the map sample is on: the ports come from it. CertScan *bool `yaml:"cert_scan"` // CertFiles are globs of certificate files the root helper reads for // the inventory (step 3): absent, certfiles.Defaults; [] reads none. // Only the certificates come back, never a key. CertFiles *[]string `yaml:"cert_files"` // CertStores are the machine's certificate stores a Windows agent // reads for the inventory (LocalMachine\): absent, // certfiles.StoreDefaults (My, WebHosting); [] reads none. Only the // certificates, never a key. Ignored elsewhere. CertStores *[]string `yaml:"cert_stores"` // HyperV: false stops a Windows agent reading the host's Hyper-V VMs // (docs/windows.md, step 8). Default true; nothing is read, and // nothing said, where Hyper-V isn't installed. Ignored elsewhere. HyperV *bool `yaml:"hyperv"` // EventLog: false stops a Windows agent reading the errors and // criticals in System and Application (docs/windows.md, step 7). // Default true. Ignored elsewhere. EventLog *bool `yaml:"event_log"` // CertReload lists the units obs-beacond may ask this agent to reload // when one serves an older certificate than its renewed file // (docs/cert-rotation.md). The root helper reads it from this file // too. Absent: nothing is ever reloaded. CertReload []report.ReloadRule `yaml:"cert_reload"` // Who watches Cairn OBS: alert through this host's own channels when // obs-beacond is gone. Set on hosts other than obs-beacond's. WatchServer *watch.Config `yaml:"watch_server"` // On one node of a Proxmox VE cluster: read the whole cluster's VMs, // containers and nodes with a PVEAuditor token (docs/cluster-watcher.md). Proxmox *proxmox.Config `yaml:"proxmox"` // A vCenter this host can reach: read its VMs, ESXi hosts, datastores // and alarms as a Read-only user (docs/cluster-watcher.md). VSphere *vsphere.Config `yaml:"vsphere"` Nutanix *nutanix.Config `yaml:"nutanix"` // Pure is the FlashArrays and FlashBlades this host reads // (docs/pure.md): any number. Pure []purerest.Config `yaml:"pure"` // IBMi is the IBM i partitions this host reads through Mapepire // (docs/ibmi.md): any number. IBMi []ibmi.Config `yaml:"ibmi"` // HMC is the IBM Hardware Management Consoles this host reads: their // Power servers and partitions, as an hmcviewer user (docs/ibmi.md, // step 4). Any number; one of a redundant pair is enough. HMC []hmc.Config `yaml:"hmc"` } func main() { log.SetFlags(0) if len(os.Args) > 1 && os.Args[1] == "scan" { if err := runScan(os.Args[2:]); err != nil { log.Fatalf("obs-beacon-agent scan: %v", err) } return } if len(os.Args) > 1 && os.Args[1] == "join" { if err := runJoin(os.Args[2:]); err != nil { log.Fatalf("obs-beacon-agent join: %v", err) } return } if len(os.Args) > 1 && os.Args[1] == "cluster" { if err := runCluster(os.Args[2:]); err != nil { log.Fatalf("obs-beacon-agent cluster: %v", err) } return } if len(os.Args) > 1 && os.Args[1] == "service" { if err := runService(os.Args[2:]); err != nil { log.Fatalf("obs-beacon-agent service: %v", err) } return } if len(os.Args) > 1 && os.Args[1] == "helper" { if err := runHelper(os.Args[2:]); err != nil { log.Fatalf("obs-beacon-agent helper: %v", err) } return } inService := serviceLogging() // Windows: started by the Service Control Manager cfgPath := flag.String("config", defaultConfig, "config file") showVersion := flag.Bool("version", false, "print the version and exit") flag.Parse() if *showVersion { fmt.Println(version.Version) return } cfg, err := loadConfig(*cfgPath) if err != nil { log.Fatalf("obs-beacon-agent: %v", err) } if inService { if err := runAsService(cfg); err != nil { log.Fatalf("obs-beacon-agent: %v", err) } return } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() if err := run(ctx, cfg); err != nil { log.Fatalf("obs-beacon-agent: %v", err) } } func loadConfig(p string) (*config, error) { b, err := os.ReadFile(p) if err != nil { return nil, err } var c config dec := yaml.NewDecoder(bytes.NewReader(b)) dec.KnownFields(true) if err := dec.Decode(&c); err != nil { return nil, fmt.Errorf("%s: %w", p, err) } if c.Server == "" { return nil, fmt.Errorf("%s: server is required", p) } if c.StateDir == "" { c.StateDir = defaultStateDir } c.Server = strings.TrimRight(c.Server, "/") if c.Interval == 0 { c.Interval = 15 * time.Second } if c.Interval < 5*time.Second { return nil, fmt.Errorf("%s: interval must be at least 5s", p) } if err := certreload.Validate(c.CertReload); err != nil { return nil, fmt.Errorf("%s: %w", p, err) } if c.Host == "" { if c.Host, err = os.Hostname(); err != nil { return nil, err } c.Host, _, _ = strings.Cut(c.Host, ".") } return &c, nil } type agent struct { cfg *config link *agentlink.Link coll collect.Collector docker *docker.Client // sources are the names of the platforms it reads, for its reports sources []string queue *queue // the SMBIOS UUID from the helper, once it has answered machine string machineKnown bool machineAsked time.Time seen map[string]bool // problems already logged, so a broken source logs once down bool // obs-beacond unreachable since the last success jitter time.Duration // the longest wait before sending a backlog (retryJitter) watch *watch.Watch // nil unless watch_server is set logs map[string]logsSeen // by container id: the last log read, for containers in trouble checks *check.Runner units []string // systemd units obs-beacond asked for services []string // services running, for adopting in the UI servicesAt time.Time // when they were last listed; zero: never noSystemd bool // systemctl isn't there: stop asking fetches []fetch.Spec // status pages obs-beacond asked for checksHave string // version of the check list running checksWant string // version obs-beacond last asked for (X-Beacon-Checks) actionsWant bool // obs-beacond has actions waiting (X-Beacon-Actions) results []report.ActionResult // what came of actions taken, for the next report certs *tlsscan.Scanner // nil when cert_scan is off files *certfiles.Watch // nil when cert_files is [] hyperV bool // read Hyper-V's VMs (Windows) events *winevent.Watch // Windows' event log; nil elsewhere or when event_log is false sampleMap func(context.Context) (*report.Map, error) // nil: no map sample here mapAcc *report.Map // map samples since the last one sent, merged mapSent time.Time // when a map sample last went with a report scan *report.Backups // the backup scan last read, and its file's time scanTime time.Time vmPrev map[string]libvirt.Domain // the last libvirt reading, by UUID, for CPU use vmSince map[string]vmSeen // when each VM's state was first seen as it is } type vmSeen struct { word string since time.Time } // readVMs reads the host's libvirt domains through the helper: nil where // there's no libvirt, so the report has no vms field. func (a *agent) readVMs(ctx context.Context) *report.VMs { vctx, cancel := context.WithTimeout(ctx, 12*time.Second) doms, err := a.docker.VMs(vctx) cancel() switch { case errors.Is(err, libvirt.ErrAbsent): a.vmPrev = nil return nil case err != nil: a.once("vms:"+err.Error(), "libvirt: %v", err) return &report.VMs{Error: err.Error()} } recs, use := libvirt.Records(doms, a.vmPrev) a.vmPrev = make(map[string]libvirt.Domain, len(doms)) for _, d := range doms { a.vmPrev[d.UUID] = d } a.stampSince(recs) return &report.VMs{List: recs, Usage: use} } // stampSince sets each VM record's Since: the hypervisor doesn't say since // when, so it's when this agent first saw it so. func (a *agent) stampSince(recs []report.Workload) { now := time.Now().UTC().Truncate(time.Second) since := make(map[string]vmSeen, len(recs)) for i, r := range recs { s, ok := a.vmSince[r.ID] if !ok || s.word != r.Word { s = vmSeen{word: r.Word, since: now} } since[r.ID], recs[i].Since = s, s.since } a.vmSince = since } // run reports until ctx ends. func run(ctx context.Context, cfg *config) error { link, err := agentlink.New(agentlink.Config{Server: cfg.Server, CAFingerprint: cfg.CAFingerprint, TokenFile: cfg.TokenFile, StateDir: cfg.StateDir}) if err != nil { return err } if !link.TLS() { log.Printf("reporting over plain http: only safe inside a VPN; use https and ca_fingerprint") } a := &agent{cfg: cfg, link: link, seen: map[string]bool{}, jitter: retryJitter(cfg.Interval)} a.queue = openQueue(filepath.Join(cfg.StateDir, "queue"), maxQueue*cfg.Interval, time.Now()) if n := a.queue.len(); n > 0 { log.Printf("%d reports from before the restart are waiting to go", n) } if cfg.HelperSocket != "" { a.docker = docker.NewClient(cfg.HelperSocket) } if cfg.WatchServer != nil { if a.watch, err = watch.New(*cfg.WatchServer, cfg.Host, time.Now()); err != nil { return err } } a.checks = check.NewRunner(ctx) if ns, err := platformSetups(cfg); err != nil { log.Print(err) a.sources = []string{} } else { a.sources = sourceNames(ns) for _, n := range ns { go a.runPlatform(ctx, n) } } // The map sample: through the root helper on Linux, the agent's own // on Windows (localMap). The certificate scan takes its ports from it. if cfg.Map == nil || *cfg.Map { switch { case a.docker != nil: a.sampleMap = a.docker.Map case localMap != nil: a.sampleMap = localMap } } if a.sampleMap != nil && (cfg.CertScan == nil || *cfg.CertScan) { a.certs = tlsscan.New(cfg.Host) go func() { t := time.NewTicker(tlsscan.RoundEvery) defer t.Stop() for { select { case <-ctx.Done(): return case now := <-t.C: a.certs.Round(ctx, now) } } }() } if globs := certfiles.Defaults; a.docker != nil && (cfg.CertFiles == nil || len(*cfg.CertFiles) > 0) { if cfg.CertFiles != nil { globs = *cfg.CertFiles } a.files = &certfiles.Watch{} go func() { logged := "" // its own goroutine: a.once is the tick's for { fctx, cancel := context.WithTimeout(ctx, time.Minute) list, err := a.docker.CertFiles(fctx, globs) cancel() if err != nil { if err.Error() != logged { logged = err.Error() log.Printf("certificate files: %v", err) } } else { a.files.Set(list) } select { case <-ctx.Done(): return case <-time.After(certfiles.AskEvery): } } }() } if a.files == nil { a.files = watchCertStores(ctx, cfg.CertStores) } a.hyperV = runtime.GOOS == "windows" && (cfg.HyperV == nil || *cfg.HyperV) if cfg.EventLog == nil || *cfg.EventLog { a.events = watchEvents(ctx) } log.Printf("obs-beacon-agent %s: host %s, reporting to %s every %s", version.Version, cfg.Host, cfg.Server, cfg.Interval) // The first sample only primes the CPU and network counters. a.coll.Collect(&report.Report{}, time.Now()) t := time.NewTicker(cfg.Interval) defer t.Stop() for { select { case <-ctx.Done(): return nil case now := <-t.C: a.tick(ctx, now) } } } func (a *agent) once(key, format string, args ...any) { if !a.seen[key] { a.seen[key] = true log.Printf(format, args...) } } func (a *agent) tick(ctx context.Context, now time.Time) { r := report.Report{ V: report.Version, Agent: version.Version, Host: a.cfg.Host, Time: now.UTC(), Interval: int(a.cfg.Interval / time.Second), Sources: a.sources, } for _, err := range a.coll.Collect(&r, now) { a.once("collect:"+err.Error(), "collect: %v", err) } if a.docker != nil { cctx, cancel := context.WithTimeout(ctx, 5*time.Second) list, err := a.docker.Containers(cctx) cancel() switch { case errors.Is(err, docker.ErrNoDocker): // a helper on a host without Docker: no containers field case err != nil: r.Containers = &report.Containers{Error: err.Error()} a.once("docker:"+err.Error(), "containers: %v", err) default: a.attachLogs(ctx, list, now) r.Containers = &report.Containers{List: list} } // The machine's UUID doesn't change while the agent runs: asked for // until the helper answers, then kept. if !a.machineKnown && now.Sub(a.machineAsked) >= 10*time.Minute { a.machineAsked = now mctx, cancel := context.WithTimeout(ctx, 3*time.Second) u, err := a.docker.Machine(mctx) cancel() if err != nil { a.once("machine:"+err.Error(), "machine UUID: %v", err) } else { a.machine, a.machineKnown = u, true } } r.Machine = a.machine r.VMs = a.readVMs(ctx) } else if a.hyperV { r.VMs = a.readHyperV() // Windows: Hyper-V's VMs, read by the agent itself } if a.sampleMap != nil { mctx, cancel := context.WithTimeout(ctx, 8*time.Second) m, err := a.sampleMap(mctx) cancel() if err != nil { a.once("map:"+err.Error(), "map sample: %v", err) m = &report.Map{Error: err.Error()} } if r.Containers != nil { nameContainers(m, r.Containers.List) } if a.certs != nil && len(m.Listen) > 0 { a.certs.Update(m.Listen, now) } a.mapAcc = netmap.Merge(a.mapAcc, m) if now.Sub(a.mapSent) >= netmap.SendEvery-a.cfg.Interval/2 { r.Map, a.mapAcc, a.mapSent = a.mapAcc, nil, now } } if len(a.units) > 0 { uctx, cancel := context.WithTimeout(ctx, 3*time.Second) us, err := units.Read(uctx, a.units) cancel() if err != nil { a.once("units:"+err.Error(), "units: %v", err) } r.Units = us } r.Services = a.runningServices(ctx, now) for _, f := range a.fetches { fctx, cancel := context.WithTimeout(ctx, 4*time.Second) r.Fetches = append(r.Fetches, fetch.Get(fctx, f)) cancel() } if a.certs != nil { r.Certs = a.certs.Report(now) } if a.files != nil { r.CertFiles = a.files.Report(now) } if a.events != nil { r.Events = a.events.Report(now) } r.Checks = a.checks.Results() r.Backups = a.backupScan() r.Reloads = a.reloadRules() r.Actions, a.results = a.results, nil b, err := json.Marshal(&r) if err != nil { log.Printf("encode: %v", err) return } a.queue.push(now, b) _ = a.link.Maintain(ctx, now) // enroll or renew when due; logs its own problems if a.down || a.queue.len() > 1 { a.wait(ctx) } a.flush(ctx) a.queue.persist() // whatever didn't go out, to disk if a.watch != nil { if a.down { a.watch.Failed(ctx, now) } else { a.watch.Reported(ctx, now) } } if a.checksWant != a.checksHave { a.fetchChecks(ctx) } if a.actionsWant { a.actionsWant = false a.takeActions(ctx) } } // nameContainers puts container names in place of the short IDs the // helper knows them by. func nameContainers(m *report.Map, list []report.Container) { names := make(map[string]string, len(list)) for _, c := range list { if len(c.ID) >= 12 { names["container:"+c.ID[:12]] = "container:" + c.Name } } for i, l := range m.Listen { if n, ok := names[l.Owner]; ok { m.Listen[i].Owner = n } } for i, c := range m.Conns { if n, ok := names[c.Owner]; ok { m.Conns[i].Owner = n } } } // runningServices lists the host's running services every five minutes: // often enough for the adopt page, rarely enough to cost nothing. func (a *agent) runningServices(ctx context.Context, now time.Time) []string { if a.noSystemd || now.Sub(a.servicesAt) < 5*time.Minute { return a.services } a.servicesAt = now sctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() list, err := units.Running(sctx) if errors.Is(err, exec.ErrNotFound) { a.noSystemd = true a.services = nil return nil } if err != nil { a.once("services:"+err.Error(), "services: %v", err) return a.services // the last good list } a.services = list return list } // fetchChecks asks obs-beacond for this host's checks after a report's answer // said the list changed. A failure keeps the old list and tries again // after the next report. func (a *agent) fetchChecks(ctx context.Context) { if a.checksWant == "" { // obs-beacond has no checks for this host a.checks.Set(nil) a.units = nil a.fetches = nil a.checksHave = "" return } ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.cfg.Server+"/api/v1/agent/checks", nil) if err != nil { return } a.link.Authorize(req) req.Header.Set("User-Agent", "obs-beacon-agent/"+version.Version) resp, err := a.link.Client().Do(req) if err != nil { a.once("checks:"+err.Error(), "can't fetch checks: %v", err) return } defer resp.Body.Close() var list struct { Version string `json:"version"` Checks []check.Spec `json:"checks"` Units []string `json:"units"` Fetches []fetch.Spec `json:"fetches"` } if resp.StatusCode != http.StatusOK || json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&list) != nil { a.once(fmt.Sprintf("checks:%d", resp.StatusCode), "can't fetch checks: HTTP %d", resp.StatusCode) return } var good []check.Spec for _, s := range list.Checks { if err := s.Check(); err != nil { log.Printf("skipping a check obs-beacond sent: %v", err) continue } good = append(good, s) } a.checks.Set(good) a.units = list.Units a.fetches = a.fetches[:0] for _, f := range list.Fetches { if err := f.Check(); err != nil { log.Printf("skipping a status page obs-beacond sent: %v", err) continue } a.fetches = append(a.fetches, f) } a.checksHave = list.Version log.Printf("running %d checks, watching %d units, reading %d status pages (list %s)", len(good), len(list.Units), len(a.fetches), list.Version) } // retryJitter is the longest an agent waits, at random, before sending // after obs-beacond was out of reach (or with reports kept from before a // restart): when obs-beacond comes back, or a fleet's agents are upgraded // together, they come back spread over that time instead of all in the // same second. Half the interval, at most 10 s. func retryJitter(interval time.Duration) time.Duration { return min(interval/2, 10*time.Second) } // wait sleeps for a random part of a.jitter, or until ctx ends. func (a *agent) wait(ctx context.Context) { if a.jitter <= 0 { return } t := time.NewTimer(rand.N(a.jitter)) defer t.Stop() select { case <-ctx.Done(): case <-t.C: } } // flush sends queued reports oldest first and stops at the first failure. // All but the newest go as queued, so obs-beacond takes them as history // rather than as the host's state now. func (a *agent) flush(ctx context.Context) { if !a.link.Ready() { // not enrolled yet: keep everything for later a.down = true return } for a.queue.len() > 0 { body, err := a.queue.peek() if err != nil { log.Printf("dropping a queued report: %v", err) a.queue.pop() continue } err = a.send(ctx, body, a.queue.len() > 1) var reject rejected switch { case err == nil: a.queue.pop() if a.down { a.down = false log.Printf("obs-beacond reachable again") } case errors.As(err, &reject) && (reject.code == http.StatusBadRequest || reject.code == http.StatusRequestEntityTooLarge): // Sending it again can't succeed; don't let it block the queue. log.Printf("obs-beacond refused a report, dropping it: %v", err) a.queue.pop() default: if errors.As(err, &reject) && reject.code == http.StatusUnauthorized { a.link.Refused() } if !a.down { a.down = true log.Printf("can't report, keeping up to %d reports: %v", maxQueue, err) } return } } } type rejected struct { code int msg string } func (r rejected) Error() string { return fmt.Sprintf("%d %s", r.code, r.msg) } func (a *agent) send(ctx context.Context, body []byte, queued bool) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.cfg.Server+"/api/v1/agent/report", bytes.NewReader(body)) if err != nil { return err } a.link.Authorize(req) req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "obs-beacon-agent/"+version.Version) if queued { req.Header.Set("X-Beacon-Queued", "1") } resp, err := a.link.Client().Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode/100 == 2 { _, _ = io.Copy(io.Discard, resp.Body) a.checksWant = resp.Header.Get("X-Beacon-Checks") a.actionsWant = a.actionsWant || resp.Header.Get("X-Beacon-Actions") != "" return nil } msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return rejected{code: resp.StatusCode, msg: strings.TrimSpace(string(msg))} } type logsSeen struct { status string at time.Time lines []string } // logLines is how much of a troubled container's log goes with a report. const logLines = 20 // troubled says whether a container's last log lines are worth sending: // it's restarting, unhealthy or dead, or it exited within the last day. func troubled(c report.Container) bool { switch { case c.State == "restarting", c.State == "dead", c.Health == "unhealthy": return true case c.State == "exited": s := c.Status return strings.Contains(s, "second") || strings.Contains(s, "minute") || strings.Contains(s, "hour") } return false } // attachLogs adds the last lines to troubled containers. A log is read // again when the container's status changes, or after two minutes, and at // most ten are read per report. func (a *agent) attachLogs(ctx context.Context, list []report.Container, now time.Time) { if a.logs == nil { a.logs = map[string]logsSeen{} } keep := map[string]bool{} reads := 0 for i := range list { c := &list[i] if !troubled(*c) { continue } keep[c.ID] = true seen, ok := a.logs[c.ID] if (!ok || seen.status != c.Status || now.Sub(seen.at) > 2*time.Minute) && reads < 10 { reads++ lctx, cancel := context.WithTimeout(ctx, 3*time.Second) lines, err := a.docker.Logs(lctx, c.ID, logLines) cancel() if err != nil { a.once("logs:"+err.Error(), "container logs: %v", err) } else { seen = logsSeen{status: c.Status, at: now, lines: lines} a.logs[c.ID] = seen } } c.Logs = seen.lines } for id := range a.logs { if !keep[id] { delete(a.logs, id) } } }