diff --git a/hack/demo-seed/README.md b/hack/demo-seed/README.md index ebd9be6..23584c1 100644 --- a/hack/demo-seed/README.md +++ b/hack/demo-seed/README.md @@ -23,6 +23,40 @@ other code. the demo, `/hack/demo-simulator`. Installed at `/etc/systemd/system/` on the demo box. +## The fleet + +Fifty hosts, shaped like an estate rather than a stack: thirty-one Linux, +eighteen Windows, and one Linux host whose agent is gone so the Agents +page has something stale to show. + +| Tier | Hosts | +|---|---| +| Edge and proxy | `lb-01/02` (HAProxy), `edge-01/02` (nginx), `proxy-01` (Squid) | +| Application | `api-01`–`04`, `worker-01`–`03`, `arm-build-01` (aarch64) | +| Data | `db-01/02` (Postgres), `mysql-01`, `cache-01/02` (Redis), `mq-01/02` (RabbitMQ), `search-01/02` (Elasticsearch) | +| Platform | `k8s-node-01`–`03` (kubelet), `ci-01` (Jenkins), `vault-01`, `ldap-01` (OpenLDAP), `dns-01` (BIND), `backup-01`, `mail-01` | +| Windows | `DC-01/02`, `IIS-01`–`03`, `WIN-SQL-01/02`, `EXCH-01/02`, `FS-01/02`, `RDS-01/02`, `WIN-APP-01/02`, `PRINT-01`, `WSUS-01`, `SCCM-01` | + +The Windows share is the point of the proportions. An enterprise looking +at this should recognise its own estate, which means Windows carrying +real services -- Active Directory, IIS, SQL Server, Exchange, file +shares, Remote Desktop, print, WSUS and SCCM -- rather than appearing +only as a Security channel on one box. + +Two hosts carry stories the alert rules fire on and must not be moved: +`worker-02`'s disk fills at 0.04 of the volume per day, which is what +`worker-disk-filling` thresholds against, and `legacy-01` checks in once +and goes quiet, which is what `agent-legacy-01-unavailable` catches. +`api-02` is the host the outage window hits. + +**Volume.** Fifty hosts generate about 316 records/minute at +`-rate-scale 1`, and the nightly reset runs at `RATE_SCALE=0.5` over a +168-hour backfill -- roughly **1.9M records per reset**, against about +0.5M when the fleet was twelve hosts. ClickHouse is untroubled by that; +what it costs is reset time and disk on the demo box. `RATE_SCALE` is the +lever if either becomes a problem, and lowering it keeps every host and +service present rather than dropping any of them. + ## Prefilled login The demo's login page comes up with the read-only `demo` account already diff --git a/hack/demo-seed/dashboards/directory-dns.json b/hack/demo-seed/dashboards/directory-dns.json new file mode 100644 index 0000000..9ce69ec --- /dev/null +++ b/hack/demo-seed/dashboards/directory-dns.json @@ -0,0 +1,109 @@ +{ + "name": "Directory and DNS", + "description": "LDAP binds, DNS resolution and the domain controllers underneath both", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "DNS queries", + "query": "service=bind | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "NXDOMAIN", + "query": "service=bind dns_rcode=NXDOMAIN | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "LDAP binds", + "query": "service=openldap event_kind=bind | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Failed binds", + "query": "service=openldap event_kind=bind_failed | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "DNS query volume by type", + "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['dns_type'] AS dns_type, count(*) AS count FROM logs WHERE service = 'bind' GROUP BY bucket, dns_type ORDER BY bucket", + "viz_type": "line", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "bucket", + "value_column": "count", + "series_column": "dns_type" + }, + "query_language": "sql", + "sort_order": 4 + }, + { + "title": "Most queried names", + "query": "service=bind | stats count by dns_name | sort -count | head 10", + "viz_type": "top_n", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "dns_name", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Bind DNs by activity", + "query": "service=openldap | stats count by bind_dn | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "bind_dn", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Failed LDAP binds", + "query": "service=openldap event_kind=bind_failed | sort -timestamp | head 25 | fields timestamp, host, bind_dn, remote_addr, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/edge-proxy.json b/hack/demo-seed/dashboards/edge-proxy.json new file mode 100644 index 0000000..944f4d2 --- /dev/null +++ b/hack/demo-seed/dashboards/edge-proxy.json @@ -0,0 +1,114 @@ +{ + "name": "Edge and proxy", + "description": "Load balancers, reverse proxies and the outbound web proxy -- what reaches the estate and what leaves it", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Balanced requests", + "query": "service=haproxy | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Backend 5xx", + "query": "service=haproxy | where status>=500 | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Proxy denials", + "query": "service=squid proxy_action=\"TCP_DENIED/403\" | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Slowest backend (avg ms)", + "query": "service=haproxy | stats avg(duration_ms) as avg_ms by backend | sort -avg_ms | head 1", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Requests per backend over time", + "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['backend'] AS backend, count(*) AS count FROM logs WHERE service = 'haproxy' GROUP BY bucket, backend ORDER BY bucket", + "viz_type": "line", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "bucket", + "value_column": "count", + "series_column": "backend" + }, + "query_language": "sql", + "sort_order": 4 + }, + { + "title": "Backend latency", + "query": "service=haproxy | stats avg(duration_ms) as avg_ms by backend | sort -avg_ms", + "viz_type": "bar", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "backend", + "value_column": "avg_ms" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Outbound destinations", + "query": "service=squid | stats count by dest_host | sort -count | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "dest_host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Status by backend", + "query": "service=haproxy | stats count by backend, status", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "backend", + "y_column": "status", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/messaging-search.json b/hack/demo-seed/dashboards/messaging-search.json new file mode 100644 index 0000000..47c3d5d --- /dev/null +++ b/hack/demo-seed/dashboards/messaging-search.json @@ -0,0 +1,109 @@ +{ + "name": "Messaging and search", + "description": "Queue depth, consumer health and the search cluster beside them", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Queue events", + "query": "service=rabbitmq | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Memory alarms", + "query": "service=rabbitmq event_kind=alarm | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Searches", + "query": "service=elasticsearch event_kind=search | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Long GC pauses", + "query": "service=elasticsearch event_kind=gc | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Queue depth over time", + "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['queue'] AS queue, max(toFloat64OrZero(attributes['queue_depth'])) AS depth FROM logs WHERE service = 'rabbitmq' GROUP BY bucket, queue ORDER BY bucket", + "viz_type": "line", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "bucket", + "value_column": "depth", + "series_column": "queue" + }, + "query_language": "sql", + "sort_order": 4 + }, + { + "title": "Deepest queues", + "query": "service=rabbitmq | stats max(queue_depth) as max_depth by queue | sort -max_depth", + "viz_type": "bar", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "queue", + "value_column": "max_depth" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Search latency by index", + "query": "service=elasticsearch event_kind=search | stats avg(duration_ms) as avg_ms by index | sort -avg_ms", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "index", + "value_column": "avg_ms" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Indexing throttles and GC", + "query": "service=elasticsearch event_kind!=search | sort -timestamp | head 25 | fields timestamp, host, index, event_kind, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/platform-kubernetes.json b/hack/demo-seed/dashboards/platform-kubernetes.json new file mode 100644 index 0000000..677ca2d --- /dev/null +++ b/hack/demo-seed/dashboards/platform-kubernetes.json @@ -0,0 +1,137 @@ +{ + "name": "Platform: Kubernetes, CI and secrets", + "description": "The tier everything else runs on -- node health, pod churn, build outcomes and secret access", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Pod events", + "query": "service=kubelet | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Probe failures", + "query": "service=kubelet event_kind=probe_failed | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Image pull failures", + "query": "service=kubelet event_kind=image_pull_failed | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Failed builds", + "query": "service=jenkins result=FAILURE | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Kubelet events over time", + "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['event_kind'] AS event_kind, count(*) AS count FROM logs WHERE service = 'kubelet' GROUP BY bucket, event_kind ORDER BY bucket", + "viz_type": "line", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "bucket", + "value_column": "count", + "series_column": "event_kind" + }, + "query_language": "sql", + "sort_order": 4 + }, + { + "title": "Busiest pods", + "query": "service=kubelet | stats count by pod | sort -count | head 10", + "viz_type": "top_n", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "pod", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Build outcomes by job", + "query": "service=jenkins | stats count by job, result", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "job", + "value_column": "count", + "series_column": "result", + "stacked": "true" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Secret reads by path", + "query": "service=vault event_kind=read | stats count by vault_path | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 13, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "vault_path", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Vault authentication failures", + "query": "service=vault event_kind=auth_failed | sort -timestamp | head 25 | fields timestamp, host, vault_path, remote_addr, message", + "viz_type": "table", + "position_x": 6, + "position_y": 13, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "Recent build failures", + "query": "service=jenkins result=FAILURE | sort -timestamp | head 25 | fields timestamp, host, job, build, duration_s", + "viz_type": "table", + "position_x": 0, + "position_y": 18, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 9 + } + ] +} diff --git a/hack/demo-seed/dashboards/windows-server-estate.json b/hack/demo-seed/dashboards/windows-server-estate.json new file mode 100644 index 0000000..662847a --- /dev/null +++ b/hack/demo-seed/dashboards/windows-server-estate.json @@ -0,0 +1,137 @@ +{ + "name": "Windows server estate", + "description": "IIS, SQL Server, Exchange and the file servers -- the Windows tier as services rather than a Security channel", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "IIS requests", + "query": "service=iis | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "IIS 5xx", + "query": "service=iis | where status>=500 | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "SQL deadlocks", + "query": "service=mssql event_kind=deadlock | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Mail bounces", + "query": "service=exchange event_kind=bounce | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "IIS traffic by site", + "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['site'] AS site, count(*) AS count FROM logs WHERE service = 'iis' GROUP BY bucket, site ORDER BY bucket", + "viz_type": "line", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "bucket", + "value_column": "count", + "series_column": "site" + }, + "query_language": "sql", + "sort_order": 4 + }, + { + "title": "IIS status mix", + "query": "service=iis | stats count by status | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "status", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "SQL Server events by database", + "query": "service=mssql | stats count by db, event_kind", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "db", + "value_column": "count", + "series_column": "event_kind", + "stacked": "true" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Busiest shares", + "query": "service=smb | stats count by share | sort -count | head 10", + "viz_type": "top_n", + "position_x": 0, + "position_y": 13, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "share", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Denied share access", + "query": "service=smb event_kind=share_denied | sort -timestamp | head 25 | fields timestamp, host, share, winevt.target_user, message", + "viz_type": "table", + "position_x": 6, + "position_y": 13, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "SQL login failures and deadlocks", + "query": "service=mssql event_kind!=login | sort -timestamp | head 25 | fields timestamp, host, db, event_kind, message", + "viz_type": "table", + "position_x": 0, + "position_y": 18, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 9 + } + ] +} diff --git a/hack/demo-simulator/events.go b/hack/demo-simulator/events.go index f0bf19f..7b98a7c 100644 --- a/hack/demo-simulator/events.go +++ b/hack/demo-simulator/events.go @@ -649,7 +649,411 @@ func primaryRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.Log return smtpRecord(h, t, r, c) case "eventlog": return eventlogRecord(h, t, r, c) + case "haproxy": + return haproxyRecord(h, t, r, c) + case "mysql": + return mysqlRecord(h, t, r, c) + case "rabbitmq": + return rabbitRecord(h, t, r, c) + case "elasticsearch": + return elasticRecord(h, t, r, c) + case "kubelet": + return kubeletRecord(h, t, r, c) + case "jenkins": + return jenkinsRecord(h, t, r, c) + case "vault": + return vaultRecord(h, t, r, c) + case "openldap": + return ldapRecord(h, t, r, c) + case "bind": + return bindRecord(h, t, r, c) + case "squid": + return squidRecord(h, t, r, c) + case "backup": + return backupRecord(h, t, r, c) + case "iis": + return iisRecord(h, t, r, c) + case "mssql": + return mssqlRecord(h, t, r, c) + case "exchange": + return exchangeRecord(h, t, r, c) + case "smb": + return smbRecord(h, t, r, c) default: return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "heartbeat", nil) } } + +// --------------------------------------------------------------------- +// Enterprise service generators. +// +// Added when the demo fleet grew from twelve hosts to fifty. The first +// six services were the ones a shop runs; these are the ones an +// enterprise runs, and they exist so a visitor sees their own estate +// rather than somebody's side project -- a load balancer in front, a +// directory and DNS underneath, a queue and a search cluster beside the +// database, CI and secrets off to one side, and a Windows tier that is a +// real tier rather than one box. +// +// Same contract as the six above: the message reads like the line the +// real daemon writes, and the attributes carry the fields inside it. +// --------------------------------------------------------------------- + +var ( + haproxyBackends = []string{"api_pool", "web_pool", "static_pool", "grpc_pool"} + + mysqlStatements = []struct { + kind, table, sql string + msMin, msMax int + }{ + {"select", "orders", "SELECT id, total, status FROM orders WHERE customer_id = ? ORDER BY created_at DESC LIMIT 50", 2, 40}, + {"select", "inventory", "SELECT sku, on_hand FROM inventory WHERE warehouse_id = ?", 1, 25}, + {"update", "inventory", "UPDATE inventory SET on_hand = on_hand - ? WHERE sku = ?", 3, 60}, + {"insert", "audit_log", "INSERT INTO audit_log (actor, action, target) VALUES (?, ?, ?)", 1, 12}, + {"select", "reports", "SELECT DATE(created_at) d, SUM(total) FROM orders GROUP BY d ORDER BY d DESC", 400, 4200}, + } + + esIndices = []string{"logs-app-000042", "logs-app-000043", "catalogue-v7", "customers-v3"} + + k8sPods = []string{ + "checkout-7d9f8b6c4-x2k9p", "catalogue-5c8b7d9f6-m4n2q", "cart-6b9d8c7f5-t8w3r", + "payments-8f7c6b5d4-j1h5g", "search-api-9d8c7b6a5-p9l2k", + } + + jenkinsJobs = []string{"shop-api/main", "shop-web/main", "platform-terraform/apply", "agent-rust/release", "nightly-integration"} + + ldapBinds = []string{ + "uid=svc_sync,ou=services,dc=shop,dc=example", "uid=jcoffey,ou=people,dc=shop,dc=example", + "uid=backup,ou=services,dc=shop,dc=example", "cn=admin,dc=shop,dc=example", + } + + dnsQueries = []struct{ name, qtype string }{ + {"api.shop.example", "A"}, {"cdn.shop.example", "AAAA"}, {"shop.example", "MX"}, + {"_ldap._tcp.shop.example", "SRV"}, {"checkout.shop.example", "A"}, {"unknown.shop.example", "A"}, + } + + vaultPaths = []string{"secret/data/shop/db", "secret/data/shop/stripe", "pki/issue/internal", "auth/kubernetes/login"} + + iisSites = []string{"Default Web Site", "ShopIntranet", "ReportingPortal"} + + mssqlDatabases = []string{"ShopERP", "ShopWarehouse", "ReportingDW", "msdb"} + + shares = []string{"\\\\FS-01\\Finance", "\\\\FS-01\\Engineering", "\\\\FS-02\\Archive", "\\\\FS-02\\Profiles"} +) + +// proxyStatus is the status code a proxy tier returns: the edge and IIS +// both mirror whatever the API behind them is doing, the same way +// nginxRecord does, so a 5xx panel agrees across all three tiers rather +// than showing an outage at one hop and health at the next. +func proxyStatus(r *rand.Rand, c conditions, slow bool) int { + errRate := c.apiErrorRate + if errRate == 0 { + errRate = 0.008 + } + if slow { + errRate *= 1.6 + } + switch { + case r.Float64() < errRate: + return pick(r, []int{502, 503, 504}) + case r.Float64() < 0.04: + return pick(r, []int{404, 401, 403, 429}) + case r.Float64() < 0.05: + return 301 + } + return 200 +} + +func haproxyRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + method, path, route, slow := pickRoute(r) + backend := pick(r, haproxyBackends) + status := proxyStatus(r, c, slow) + // HAProxy's timing quintuple is the thing operators actually read. + tq, tw, tc := r.Intn(4), r.Intn(3), r.Intn(6) + tr := int(float64(8+r.Intn(180)) * c.latencyMult) + tt := tq + tw + tc + tr + srvConn, beConn := r.Intn(12), r.Intn(40) + sev := logsv1.Severity_SEVERITY_INFO + if status >= 500 { + sev = logsv1.Severity_SEVERITY_ERROR + } + src := pick(r, clientIPs) + return newRecord(h, "haproxy", t, sev, + fmt.Sprintf("%s:%d [%s] https-in~ %s/srv%d %d/%d/%d/%d/%d %d %d - - ---- %d/%d/%d/%d/0 0/0 \"%s %s HTTP/2.0\"", + src, 40000+r.Intn(20000), t.Format("02/Jan/2006:15:04:05.000"), + backend, 1+r.Intn(4), tq, tw, tc, tr, tt, status, 200+r.Intn(9000), + beConn, srvConn, r.Intn(3), r.Intn(2), method, path), + map[string]string{ + "backend": backend, "status": strconv.Itoa(status), "method": method, + "route": route, "duration_ms": strconv.Itoa(tt), "remote_addr": src, + "be_conn": strconv.Itoa(beConn), "srv_conn": strconv.Itoa(srvConn), + }) +} + +func mysqlRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + if r.Float64() < 0.15 { + return newRecord(h, "mysql", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("[Note] Aborted connection %d to db: 'shop' user: 'shop_app' host: '%s' (Got timeout reading communication packets)", + 100000+r.Intn(900000), pick(r, clientIPs)), + map[string]string{"db": "shop", "query_kind": "connection_error"}) + } + s := pick(r, mysqlStatements) + dur := float64(s.msMin+r.Intn(s.msMax-s.msMin)) * c.latencyMult + sev := logsv1.Severity_SEVERITY_DEBUG + if dur > 1000 { + sev = logsv1.Severity_SEVERITY_WARN + } + return newRecord(h, "mysql", t, sev, + fmt.Sprintf("# Query_time: %.6f Lock_time: %.6f Rows_sent: %d Rows_examined: %d\n%s", + dur/1000, r.Float64()/500, r.Intn(200), r.Intn(20000), s.sql), + map[string]string{ + "db": "shop", "db_user": "shop_app", "query_kind": s.kind, "table": s.table, + "duration_ms": fmt.Sprintf("%.1f", dur), "rows": strconv.Itoa(r.Intn(200)), + }) +} + +func rabbitRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + queue := pick(r, []string{"orders.process", "mail.outbound", "search.index", "invoice.render"}) + depth := r.Intn(400) + // A queue backs up when the workers draining it are failing, which is + // what the outage window already models -- rather than inventing a + // second condition that says the same thing. + if c.jobFailureRate > 0.2 { + depth += 2000 + r.Intn(6000) + } + switch n := r.Intn(10); { + case n < 6: + return newRecord(h, "rabbitmq", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("accepting AMQP connection <0.%d.0> (%s:%d -> 10.0.4.21:5672)", 1000+r.Intn(9000), pick(r, clientIPs), 40000+r.Intn(20000)), + map[string]string{"queue": queue, "queue_depth": strconv.Itoa(depth), "event_kind": "connect"}) + case n < 9: + return newRecord(h, "rabbitmq", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Queue '%s' in vhost '/': %d messages ready, %d unacknowledged", queue, depth, r.Intn(40)), + map[string]string{"queue": queue, "queue_depth": strconv.Itoa(depth), "event_kind": "depth"}) + default: + return newRecord(h, "rabbitmq", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("memory resource limit alarm set on node rabbit@%s -- publishers will be blocked", h.name), + map[string]string{"queue": queue, "queue_depth": strconv.Itoa(depth), "event_kind": "alarm"}) + } +} + +func elasticRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + idx := pick(r, esIndices) + took := int(float64(4+r.Intn(300)) * c.latencyMult) + switch n := r.Intn(10); { + case n < 7: + return newRecord(h, "elasticsearch", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("[%s] search completed in %dms, hits=%d, shards=[3 total, 3 successful]", idx, took, r.Intn(4000)), + map[string]string{"index": idx, "duration_ms": strconv.Itoa(took), "event_kind": "search"}) + case n < 9: + return newRecord(h, "elasticsearch", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("[%s] now throttling indexing: numMergesInFlight=%d, maxNumMerges=%d", idx, 4+r.Intn(4), 5), + map[string]string{"index": idx, "event_kind": "throttle"}) + default: + return newRecord(h, "elasticsearch", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("[gc][old][%d][%d] duration [%dms], collections [1]/[%ds]", r.Intn(90000), r.Intn(400), 400+r.Intn(2200), 1+r.Intn(3)), + map[string]string{"index": idx, "event_kind": "gc"}) + } +} + +func kubeletRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + pod := pick(r, k8sPods) + ns := "shop" + switch n := r.Intn(12); { + case n < 6: + return newRecord(h, "kubelet", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("SyncLoop (PLEG): %q, event: &pod.LifecycleEvent{ID:%q, Type:\"ContainerStarted\"}", ns+"/"+pod, pod), + map[string]string{"pod": pod, "namespace": ns, "event_kind": "sync"}) + case n < 9: + return newRecord(h, "kubelet", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Probe succeeded for pod %q container %q: HTTP 200 in %dms", ns+"/"+pod, strings.Split(pod, "-")[0], 2+r.Intn(40)), + map[string]string{"pod": pod, "namespace": ns, "event_kind": "probe"}) + case n < 11: + return newRecord(h, "kubelet", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("Readiness probe failed for pod %q: Get \"http://10.42.0.%d:8080/healthz\": context deadline exceeded", ns+"/"+pod, r.Intn(250)), + map[string]string{"pod": pod, "namespace": ns, "event_kind": "probe_failed"}) + default: + return newRecord(h, "kubelet", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("Failed to pull image \"registry.shop.example/%s:v1.4.%d\": rpc error: code = DeadlineExceeded", strings.Split(pod, "-")[0], r.Intn(40)), + map[string]string{"pod": pod, "namespace": ns, "event_kind": "image_pull_failed"}) + } +} + +func jenkinsRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + job := pick(r, jenkinsJobs) + build := 400 + r.Intn(900) + dur := 40 + r.Intn(900) + if r.Float64() < 0.12 { + return newRecord(h, "jenkins", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("%s #%d completed: FAILURE after %ds", job, build, dur), + map[string]string{"job": job, "build": strconv.Itoa(build), "result": "FAILURE", "duration_s": strconv.Itoa(dur)}) + } + return newRecord(h, "jenkins", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("%s #%d completed: SUCCESS after %ds", job, build, dur), + map[string]string{"job": job, "build": strconv.Itoa(build), "result": "SUCCESS", "duration_s": strconv.Itoa(dur)}) +} + +func vaultRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + path := pick(r, vaultPaths) + if c.bruteForce && r.Float64() < 0.25 { + return newRecord(h, "vault", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("authentication failed: path=%s remote_address=%s error=\"permission denied\"", path, pick(r, attackerIPs)), + map[string]string{"vault_path": path, "event_kind": "auth_failed", "remote_addr": pick(r, attackerIPs)}) + } + return newRecord(h, "vault", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("request: operation=read path=%s remote_address=10.0.2.%d ttl=%dm", path, r.Intn(250), 30+r.Intn(700)), + map[string]string{"vault_path": path, "event_kind": "read"}) +} + +func ldapRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + dn := pick(r, ldapBinds) + if c.bruteForce && r.Float64() < 0.3 { + return newRecord(h, "openldap", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("conn=%d op=1 RESULT tag=97 err=49 text=Invalid credentials dn=%q", 10000+r.Intn(90000), dn), + map[string]string{"bind_dn": dn, "ldap_err": "49", "event_kind": "bind_failed", "remote_addr": pick(r, attackerIPs)}) + } + switch r.Intn(3) { + case 0: + return newRecord(h, "openldap", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("conn=%d op=0 BIND dn=%q method=128 mech=SIMPLE ssf=256", 10000+r.Intn(90000), dn), + map[string]string{"bind_dn": dn, "event_kind": "bind"}) + default: + return newRecord(h, "openldap", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("conn=%d op=2 SEARCH RESULT tag=101 err=0 nentries=%d text=", 10000+r.Intn(90000), r.Intn(60)), + map[string]string{"bind_dn": dn, "event_kind": "search"}) + } +} + +func bindRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + q := dnsQueries[r.Intn(len(dnsQueries))] + src := pick(r, clientIPs) + if q.name == "unknown.shop.example" { + return newRecord(h, "bind", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("client @0x%x %s#%d (%s): query: %s IN %s + (10.0.5.2) NXDOMAIN", r.Int63(), src, 30000+r.Intn(30000), q.name, q.name, q.qtype), + map[string]string{"dns_name": q.name, "dns_type": q.qtype, "dns_rcode": "NXDOMAIN", "remote_addr": src}) + } + return newRecord(h, "bind", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("client @0x%x %s#%d (%s): query: %s IN %s + (10.0.5.2)", r.Int63(), src, 30000+r.Intn(30000), q.name, q.name, q.qtype), + map[string]string{"dns_name": q.name, "dns_type": q.qtype, "dns_rcode": "NOERROR", "remote_addr": src}) +} + +func squidRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + sites := []string{"github.com", "registry.npmjs.org", "download.docker.com", "ubuntu.com", "ads.example.net"} + site := pick(r, sites) + action, status := "TCP_MISS/200", 200 + sev := logsv1.Severity_SEVERITY_INFO + if site == "ads.example.net" { + action, status, sev = "TCP_DENIED/403", 403, logsv1.Severity_SEVERITY_WARN + } + src := "10.0.3." + strconv.Itoa(r.Intn(250)) + return newRecord(h, "squid", t, sev, + fmt.Sprintf("%d.%03d %6d %s %s %d CONNECT %s:443 - HIER_DIRECT/%s -", + t.Unix(), r.Intn(1000), r.Intn(4000), src, action, 500+r.Intn(90000), site, site), + map[string]string{"proxy_action": action, "status": strconv.Itoa(status), "dest_host": site, "remote_addr": src}) +} + +func backupRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + job := pick(r, []string{"nightly-fileserver", "nightly-postgres", "weekly-archive", "hourly-mssql"}) + gb := 4 + r.Float64()*90 + if r.Float64() < 0.1 { + return newRecord(h, "backup", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("Job %s terminated with errors: cannot open source \"%s\": permission denied", job, pick(r, shares)), + map[string]string{"backup_job": job, "result": "error", "event_kind": "backup"}) + } + return newRecord(h, "backup", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Job %s OK: %.1f GB written in %d min, dedup ratio %.2fx", job, gb, 8+r.Intn(90), 1.8+r.Float64()*3), + map[string]string{"backup_job": job, "result": "ok", "bytes_gb": fmt.Sprintf("%.1f", gb), "event_kind": "backup"}) +} + +func iisRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + method, path, route, slow := pickRoute(r) + site := pick(r, iisSites) + status := proxyStatus(r, c, slow) + took := int(float64(6+r.Intn(400)) * c.latencyMult) + src := pick(r, clientIPs) + sev := logsv1.Severity_SEVERITY_INFO + if status >= 500 { + sev = logsv1.Severity_SEVERITY_ERROR + } + // W3C extended log format, which is what IIS actually writes. + return newRecord(h, "iis", t, sev, + fmt.Sprintf("%s %s %s %s - 443 - %s HTTP/2.0 %s - - %s %d 0 0 %d", + t.Format("2006-01-02"), t.Format("15:04:05"), h.ipv4, method, src, + pick(r, userAgents), path, status, took), + map[string]string{ + "site": site, "status": strconv.Itoa(status), "method": method, + "route": route, "duration_ms": strconv.Itoa(took), "remote_addr": src, + }) +} + +func mssqlRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + db := pick(r, mssqlDatabases) + switch n := r.Intn(12); { + case n < 5: + return newRecord(h, "mssql", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Login succeeded for user 'SHOP\\svc_erp'. Connection made using Windows authentication. [CLIENT: %s]", pick(r, clientIPs)), + map[string]string{"db": db, "event_kind": "login"}) + case n < 8: + ms := int(float64(200+r.Intn(9000)) * c.latencyMult) + sev := logsv1.Severity_SEVERITY_INFO + if ms > 5000 { + sev = logsv1.Severity_SEVERITY_WARN + } + return newRecord(h, "mssql", t, sev, + fmt.Sprintf("SQL Server has encountered %d occurrence(s) of I/O requests taking longer than %d seconds to complete on file [F:\\Data\\%s.mdf]", 1+r.Intn(4), 15, db), + map[string]string{"db": db, "duration_ms": strconv.Itoa(ms), "event_kind": "io_stall"}) + case n < 10: + return newRecord(h, "mssql", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Log was backed up. Database: %s, creation date(time): 2026/01/14(03:11:02), first LSN: %d:%d:1", db, 40+r.Intn(60), r.Intn(90000)), + map[string]string{"db": db, "event_kind": "log_backup"}) + case n < 11: + return newRecord(h, "mssql", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("Transaction (Process ID %d) was deadlocked on lock resources with another process and has been chosen as the deadlock victim in database %s", 50+r.Intn(200), db), + map[string]string{"db": db, "event_kind": "deadlock"}) + default: + return newRecord(h, "mssql", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("Login failed for user 'SHOP\\svc_report'. Reason: Password did not match that for the login provided. [CLIENT: %s]", pick(r, attackerIPs)), + map[string]string{"db": db, "event_kind": "login_failed", "remote_addr": pick(r, attackerIPs)}) + } +} + +func exchangeRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + box := pick(r, []string{"jcoffey@shop.example", "orders@shop.example", "support@shop.example", "hr@shop.example"}) + switch n := r.Intn(10); { + case n < 5: + return newRecord(h, "exchange", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("RECEIVE SMTP 250 2.6.0 message queued for delivery to %s, size %d bytes", box, 2000+r.Intn(200000)), + map[string]string{"mailbox": box, "event_kind": "receive"}) + case n < 8: + return newRecord(h, "exchange", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("SEND SMTP 250 2.0.0 delivered to remote host for %s in %dms", box, 40+r.Intn(2000)), + map[string]string{"mailbox": box, "event_kind": "send"}) + case n < 9: + return newRecord(h, "exchange", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("AGENT SpamFilter rejected message for %s: SCL 7 above threshold", box), + map[string]string{"mailbox": box, "event_kind": "spam_reject"}) + default: + return newRecord(h, "exchange", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("FAIL 550 5.1.1 User unknown: no mailbox by that name at shop.example (recipient %s)", box), + map[string]string{"mailbox": box, "event_kind": "bounce"}) + } +} + +func smbRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + share := pick(r, shares) + user := pick(r, []string{"SHOP\\jcoffey", "SHOP\\svc_backup", "SHOP\\finance_ro", "SHOP\\eng_rw"}) + switch n := r.Intn(10); { + case n < 7: + return newRecord(h, "smb", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("A network share object was accessed. Share Name: %s Account Name: %s Access: ReadData", share, user), + map[string]string{"share": share, "winevt.target_user": user, "event_kind": "share_access"}) + case n < 9: + return newRecord(h, "smb", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("File written on share %s by %s (%d bytes)", share, user, 1000+r.Intn(9000000)), + map[string]string{"share": share, "winevt.target_user": user, "event_kind": "share_write"}) + default: + return newRecord(h, "smb", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("A network share object access was denied. Share Name: %s Account Name: %s", share, user), + map[string]string{"share": share, "winevt.target_user": user, "event_kind": "share_denied"}) + } +} diff --git a/hack/demo-simulator/fleet.go b/hack/demo-simulator/fleet.go index ab15293..db4b0ee 100644 --- a/hack/demo-simulator/fleet.go +++ b/hack/demo-simulator/fleet.go @@ -1,5 +1,7 @@ package main +import "strings" + // The synthetic fleet the demo deployment pretends to be monitoring: a // small e-commerce shop's infrastructure. Every host here is fictional, // but the shape is deliberately realistic -- an edge/nginx tier, a @@ -71,6 +73,37 @@ type host struct { const agentVersion = "0.6.2" var fleet = []host{ + // Fifty hosts, shaped like an estate rather than a stack: a proxy + // tier in front, application and worker tiers behind it, the data + // services they lean on, a platform tier (Kubernetes, CI, secrets, + // directory, DNS, backup), and a Windows tier that is a tier rather + // than a token box. Thirty-one Linux, eighteen Windows, and one + // Linux host whose agent is gone. + // + // The proportions matter more than the count: an enterprise looking + // at this should recognise its own estate, which means Windows + // carrying real services (AD, IIS, SQL Server, Exchange, file, RDS, + // print, WSUS/SCCM) instead of appearing only as a Security channel. + { + name: "lb-01", service: "haproxy", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 60 << 30, + ipv4: "10.0.1.5", ipv6: "2600:3c02::f03c:94ff:fe1a:2001", + cpuBase: 26, memFrac: 0.36, diskFrac: 0.28, + eventsPerMin: 14, systemPerMin: 0.7, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/haproxy.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "lb-02", service: "haproxy", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 60 << 30, + ipv4: "10.0.1.6", ipv6: "2600:3c02::f03c:94ff:fe1a:2002", + cpuBase: 24, memFrac: 0.34, diskFrac: 0.27, + eventsPerMin: 13, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/haproxy.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, { name: "edge-01", service: "nginx", os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", @@ -91,124 +124,460 @@ var fleet = []host{ agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/nginx/access.log", batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, + { + name: "proxy-01", service: "squid", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 2, memTotal: 4 << 30, diskTot: 80 << 30, + ipv4: "10.0.3.10", ipv6: "2600:3c02::f03c:94ff:fe1a:3001", + cpuBase: 12, memFrac: 0.29, diskFrac: 0.44, + eventsPerMin: 6, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/squid/access.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, { name: "api-01", service: "api", - os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", - cores: 8, memTotal: 16 << 30, diskTot: 160 << 30, - ipv4: "10.0.2.21", ipv6: "2600:3c02::f03c:94ff:fe1a:2101", - cpuBase: 34, memFrac: 0.52, diskFrac: 0.29, - eventsPerMin: 10, systemPerMin: 0.5, + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.2.21", ipv6: "2600:3c02::f03c:94ff:fe1a:2201", + cpuBase: 44, memFrac: 0.62, diskFrac: 0.41, + eventsPerMin: 22, systemPerMin: 0.8, agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service", - batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000, + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { name: "api-02", service: "api", - os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", - cores: 8, memTotal: 16 << 30, diskTot: 160 << 30, - ipv4: "10.0.2.22", ipv6: "2600:3c02::f03c:94ff:fe1a:2102", - cpuBase: 37, memFrac: 0.57, diskFrac: 0.31, - eventsPerMin: 10, systemPerMin: 0.5, + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.2.22", ipv6: "2600:3c02::f03c:94ff:fe1a:2202", + cpuBase: 47, memFrac: 0.62, diskFrac: 0.41, + eventsPerMin: 24, systemPerMin: 0.8, agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service", - batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000, + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { name: "api-03", service: "api", - // One host deliberately a release behind, so the Agents page's - // agent_version column shows a fleet that isn't uniformly - // upgraded -- the normal state of any real fleet. - os: "Debian GNU/Linux 12 (bookworm)", kernel: "6.1.0-25-amd64", arch: "x86_64", - cores: 4, memTotal: 8 << 30, diskTot: 160 << 30, - ipv4: "10.0.2.23", ipv6: "2600:3c02::f03c:94ff:fe1a:2103", - cpuBase: 41, memFrac: 0.61, diskFrac: 0.44, - eventsPerMin: 9, systemPerMin: 0.5, - agentVersion: "0.5.4", sourceKind: "journald", sourceDetail: "unit=shop-api.service", - batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000, + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.2.23", ipv6: "2600:3c02::f03c:94ff:fe1a:2203", + cpuBase: 41, memFrac: 0.62, diskFrac: 0.41, + eventsPerMin: 21, systemPerMin: 0.8, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "api-04", service: "api", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.2.24", ipv6: "2600:3c02::f03c:94ff:fe1a:2204", + cpuBase: 39, memFrac: 0.62, diskFrac: 0.41, + eventsPerMin: 19, systemPerMin: 0.8, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { name: "worker-01", service: "worker", - os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", cores: 4, memTotal: 8 << 30, diskTot: 200 << 30, - ipv4: "10.0.3.31", ipv6: "2600:3c02::f03c:94ff:fe1a:3101", - cpuBase: 46, memFrac: 0.63, diskFrac: 0.4, - eventsPerMin: 4.5, systemPerMin: 0.4, + ipv4: "10.0.2.41", ipv6: "2600:3c02::f03c:94ff:fe1a:2241", + cpuBase: 31, memFrac: 0.55, diskFrac: 0.48, + eventsPerMin: 9, systemPerMin: 0.7, agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service", - batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { name: "worker-02", service: "worker", - os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", cores: 4, memTotal: 8 << 30, diskTot: 200 << 30, - ipv4: "10.0.3.32", ipv6: "2600:3c02::f03c:94ff:fe1a:3102", + ipv4: "10.0.2.42", ipv6: "2600:3c02::f03c:94ff:fe1a:2242", cpuBase: 52, memFrac: 0.71, diskFrac: 0.62, - // The one host with a real, visible trend: ~4 points of disk a - // day, so a 7-day backfill window ends with it close to full and - // the "Disk filling up" alert rule has something true to fire on. + // The disk-filling story the worker-disk-filling alert rule fires + // on. It has to stay on this host and at this rate: the rule names + // worker-02 and thresholds on 175 GiB of its 200. diskGrowthPerDay: 0.04, - eventsPerMin: 4.5, systemPerMin: 0.4, + eventsPerMin: 8, systemPerMin: 0.7, agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service", - batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { - name: "db-01", service: "postgres", - os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", - cores: 8, memTotal: 32 << 30, diskTot: 500 << 30, - ipv4: "10.0.4.41", ipv6: "2600:3c02::f03c:94ff:fe1a:4101", - cpuBase: 28, memFrac: 0.74, diskFrac: 0.51, - eventsPerMin: 6, systemPerMin: 0.4, - agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/postgresql/postgresql-17-main.log", - batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, - }, - { - name: "cache-01", service: "redis", - os: "Ubuntu 22.04.5 LTS", kernel: "5.15.0-118-generic", arch: "x86_64", - cores: 2, memTotal: 8 << 30, diskTot: 50 << 30, - ipv4: "10.0.4.51", ipv6: "2600:3c02::f03c:94ff:fe1a:5101", - cpuBase: 11, memFrac: 0.58, diskFrac: 0.18, - eventsPerMin: 2, systemPerMin: 0.3, - agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=redis-server.service", - batchMax: 500, batchFlushMS: 10000, heartbeatMS: 60000, - }, - { - name: "mail-01", service: "smtp", - os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", - cores: 2, memTotal: 4 << 30, diskTot: 250 << 30, - ipv4: "198.51.100.25", ipv6: "2600:3c06::2000:7dff:fe55:2501", - cpuBase: 14, memFrac: 0.46, diskFrac: 0.57, - eventsPerMin: 6, systemPerMin: 1.2, // internet-facing: more scan/ssh noise than an internal host - agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/opt/stalwart/logs/current.log", + name: "worker-03", service: "worker", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 200 << 30, + ipv4: "10.0.2.43", ipv6: "2600:3c02::f03c:94ff:fe1a:2243", + cpuBase: 33, memFrac: 0.55, diskFrac: 0.48, + eventsPerMin: 9, systemPerMin: 0.7, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service", batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { name: "arm-build-01", service: "worker", - // The fleet's one non-x86 host, so `stats count by arch`-style - // questions and the Hosts page's Architecture row have more than - // one answer in them. - os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "aarch64", - cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, - ipv4: "10.0.5.61", ipv6: "2600:3c02::f03c:94ff:fe1a:6101", - cpuBase: 63, memFrac: 0.55, diskFrac: 0.47, - eventsPerMin: 3, systemPerMin: 0.3, - agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=buildkite-agent.service", - batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + os: "Debian GNU/Linux 12 (bookworm)", kernel: "6.1.0-27-arm64", arch: "aarch64", + cores: 8, memTotal: 16 << 30, diskTot: 300 << 30, + ipv4: "10.0.2.51", ipv6: "2600:3c02::f03c:94ff:fe1a:2251", + cpuBase: 55, memFrac: 0.61, diskFrac: 0.52, + eventsPerMin: 7, systemPerMin: 0.8, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "db-01", service: "postgres", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 1000 << 30, + ipv4: "10.0.4.11", ipv6: "2600:3c02::f03c:94ff:fe1a:4001", + cpuBase: 38, memFrac: 0.71, diskFrac: 0.58, + eventsPerMin: 18, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=postgresql@16-main.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "db-02", service: "postgres", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 1000 << 30, + ipv4: "10.0.4.12", ipv6: "2600:3c02::f03c:94ff:fe1a:4002", + cpuBase: 22, memFrac: 0.66, diskFrac: 0.57, + eventsPerMin: 9, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=postgresql@16-main.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "mysql-01", service: "mysql", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 600 << 30, + ipv4: "10.0.4.13", ipv6: "2600:3c02::f03c:94ff:fe1a:4003", + cpuBase: 29, memFrac: 0.68, diskFrac: 0.49, + eventsPerMin: 11, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=mysql.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "cache-01", service: "redis", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 60 << 30, + ipv4: "10.0.4.21", ipv6: "2600:3c02::f03c:94ff:fe1a:4011", + cpuBase: 9, memFrac: 0.44, diskFrac: 0.19, + eventsPerMin: 7, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=redis-server.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "cache-02", service: "redis", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 60 << 30, + ipv4: "10.0.4.22", ipv6: "2600:3c02::f03c:94ff:fe1a:4012", + cpuBase: 8, memFrac: 0.42, diskFrac: 0.18, + eventsPerMin: 6, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=redis-server.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "mq-01", service: "rabbitmq", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.4.31", ipv6: "2600:3c02::f03c:94ff:fe1a:4021", + cpuBase: 17, memFrac: 0.51, diskFrac: 0.26, + eventsPerMin: 10, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=rabbitmq-server.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "mq-02", service: "rabbitmq", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.4.32", ipv6: "2600:3c02::f03c:94ff:fe1a:4022", + cpuBase: 15, memFrac: 0.49, diskFrac: 0.25, + eventsPerMin: 9, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=rabbitmq-server.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "search-01", service: "elasticsearch", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 800 << 30, + ipv4: "10.0.4.41", ipv6: "2600:3c02::f03c:94ff:fe1a:4031", + cpuBase: 41, memFrac: 0.79, diskFrac: 0.62, + eventsPerMin: 12, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=elasticsearch.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "search-02", service: "elasticsearch", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 800 << 30, + ipv4: "10.0.4.42", ipv6: "2600:3c02::f03c:94ff:fe1a:4032", + cpuBase: 39, memFrac: 0.77, diskFrac: 0.61, + eventsPerMin: 11, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=elasticsearch.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "k8s-node-01", service: "kubelet", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 400 << 30, + ipv4: "10.0.7.11", ipv6: "2600:3c02::f03c:94ff:fe1a:7011", + cpuBase: 52, memFrac: 0.73, diskFrac: 0.44, + eventsPerMin: 16, systemPerMin: 0.9, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=kubelet.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "k8s-node-02", service: "kubelet", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 400 << 30, + ipv4: "10.0.7.12", ipv6: "2600:3c02::f03c:94ff:fe1a:7012", + cpuBase: 49, memFrac: 0.73, diskFrac: 0.44, + eventsPerMin: 15, systemPerMin: 0.9, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=kubelet.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "k8s-node-03", service: "kubelet", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 400 << 30, + ipv4: "10.0.7.13", ipv6: "2600:3c02::f03c:94ff:fe1a:7013", + cpuBase: 47, memFrac: 0.73, diskFrac: 0.44, + eventsPerMin: 14, systemPerMin: 0.9, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=kubelet.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "ci-01", service: "jenkins", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 500 << 30, + ipv4: "10.0.7.21", ipv6: "2600:3c02::f03c:94ff:fe1a:7021", + cpuBase: 58, memFrac: 0.64, diskFrac: 0.69, + eventsPerMin: 8, systemPerMin: 0.7, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=jenkins.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "vault-01", service: "vault", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 2, memTotal: 8 << 30, diskTot: 40 << 30, + ipv4: "10.0.7.31", ipv6: "2600:3c02::f03c:94ff:fe1a:7031", + cpuBase: 7, memFrac: 0.31, diskFrac: 0.21, + eventsPerMin: 5, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=vault.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "ldap-01", service: "openldap", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 2, memTotal: 8 << 30, diskTot: 40 << 30, + ipv4: "10.0.7.41", ipv6: "2600:3c02::f03c:94ff:fe1a:7041", + cpuBase: 11, memFrac: 0.34, diskFrac: 0.23, + eventsPerMin: 7, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=slapd.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "dns-01", service: "bind", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 2, memTotal: 4 << 30, diskTot: 30 << 30, + ipv4: "10.0.5.2", ipv6: "2600:3c02::f03c:94ff:fe1a:5001", + cpuBase: 8, memFrac: 0.27, diskFrac: 0.17, + eventsPerMin: 12, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=named.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "backup-01", service: "backup", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 4000 << 30, + ipv4: "10.0.8.11", ipv6: "2600:3c02::f03c:94ff:fe1a:8001", + cpuBase: 19, memFrac: 0.41, diskFrac: 0.74, + diskGrowthPerDay: 0.012, + eventsPerMin: 4, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/bacula/bacula.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "mail-01", service: "smtp", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 200 << 30, + ipv4: "10.0.6.11", ipv6: "2600:3c02::f03c:94ff:fe1a:6001", + cpuBase: 16, memFrac: 0.47, diskFrac: 0.39, + eventsPerMin: 9, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=stalwart-mail.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "DC-01", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 300 << 30, + ipv4: "10.0.6.21", ipv6: "", + cpuBase: 26, memFrac: 0.61, diskFrac: 0.44, + eventsPerMin: 6, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application,Directory Service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "DC-02", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 300 << 30, + ipv4: "10.0.6.22", ipv6: "", + cpuBase: 24, memFrac: 0.59, diskFrac: 0.43, + eventsPerMin: 5.5, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application,Directory Service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "IIS-01", service: "iis", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 250 << 30, + ipv4: "10.0.6.31", ipv6: "", + cpuBase: 34, memFrac: 0.58, diskFrac: 0.47, + eventsPerMin: 12, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "IIS-02", service: "iis", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 250 << 30, + ipv4: "10.0.6.32", ipv6: "", + cpuBase: 32, memFrac: 0.56, diskFrac: 0.46, + eventsPerMin: 11, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "IIS-03", service: "iis", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 250 << 30, + ipv4: "10.0.6.33", ipv6: "", + cpuBase: 30, memFrac: 0.55, diskFrac: 0.45, + eventsPerMin: 10, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "WIN-SQL-01", service: "mssql", + os: "Windows Server 2019 Standard", kernel: "10.0.17763", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 2000 << 30, + ipv4: "10.0.6.72", ipv6: "", + cpuBase: 43, memFrac: 0.81, diskFrac: 0.68, + eventsPerMin: 9, systemPerMin: 0, + agentVersion: "0.5.4", sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "WIN-SQL-02", service: "mssql", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 2000 << 30, + ipv4: "10.0.6.73", ipv6: "", + cpuBase: 39, memFrac: 0.79, diskFrac: 0.66, + eventsPerMin: 8, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "EXCH-01", service: "exchange", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 1500 << 30, + ipv4: "10.0.6.41", ipv6: "", + cpuBase: 37, memFrac: 0.74, diskFrac: 0.71, + eventsPerMin: 10, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "EXCH-02", service: "exchange", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 1500 << 30, + ipv4: "10.0.6.42", ipv6: "", + cpuBase: 34, memFrac: 0.72, diskFrac: 0.69, + eventsPerMin: 9, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "FS-01", service: "smb", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 8000 << 30, + ipv4: "10.0.6.51", ipv6: "", + cpuBase: 14, memFrac: 0.48, diskFrac: 0.79, + eventsPerMin: 8, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "FS-02", service: "smb", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 8000 << 30, + ipv4: "10.0.6.52", ipv6: "", + cpuBase: 12, memFrac: 0.46, diskFrac: 0.83, + diskGrowthPerDay: 0.008, + eventsPerMin: 7, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "RDS-01", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 400 << 30, + ipv4: "10.0.6.61", ipv6: "", + cpuBase: 48, memFrac: 0.77, diskFrac: 0.51, + eventsPerMin: 7, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application,TerminalServices", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "RDS-02", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 16, memTotal: 64 << 30, diskTot: 400 << 30, + ipv4: "10.0.6.62", ipv6: "", + cpuBase: 45, memFrac: 0.75, diskFrac: 0.49, + eventsPerMin: 6.5, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application,TerminalServices", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { name: "WIN-APP-01", service: "eventlog", os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", - cores: 4, memTotal: 16 << 30, diskTot: 250 << 30, + cores: 4, memTotal: 16 << 30, diskTot: 200 << 30, ipv4: "10.0.6.71", ipv6: "", - cpuBase: 26, memFrac: 0.64, diskFrac: 0.42, + cpuBase: 18, memFrac: 0.52, diskFrac: 0.4, eventsPerMin: 2.5, systemPerMin: 0, agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { - name: "WIN-SQL-01", service: "eventlog", - os: "Windows Server 2019 Standard", kernel: "10.0.17763", arch: "x86_64", - cores: 8, memTotal: 32 << 30, diskTot: 500 << 30, - ipv4: "10.0.6.72", ipv6: "", - cpuBase: 33, memFrac: 0.78, diskFrac: 0.66, - eventsPerMin: 2, systemPerMin: 0, - agentVersion: "0.5.4", sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + name: "WIN-APP-02", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 200 << 30, + ipv4: "10.0.6.74", ipv6: "", + cpuBase: 16, memFrac: 0.5, diskFrac: 0.38, + eventsPerMin: 2.2, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "PRINT-01", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 120 << 30, + ipv4: "10.0.6.81", ipv6: "", + cpuBase: 9, memFrac: 0.38, diskFrac: 0.31, + eventsPerMin: 3, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application,PrintService", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "WSUS-01", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 900 << 30, + ipv4: "10.0.6.91", ipv6: "", + cpuBase: 11, memFrac: 0.44, diskFrac: 0.72, + eventsPerMin: 3.5, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "SCCM-01", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 1200 << 30, + ipv4: "10.0.6.92", ipv6: "", + cpuBase: 21, memFrac: 0.57, diskFrac: 0.63, + eventsPerMin: 4, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, { @@ -224,15 +593,26 @@ var fleet = []host{ }, } +// windows reports whether this host's agent reads Windows channels +// rather than journald or a file. +// +// Decided from `os` rather than from `service`, which is what it used to +// key off. That worked while "eventlog" was the only Windows role; the +// Windows tier now runs IIS, SQL Server, Exchange and file servers, and +// keying off the role would have handed every one of them a journald +// `system` stream -- sshd and UFW lines on a Windows box, which is the +// kind of wrong that is hard to notice and impossible to unsee. +func (h *host) windows() bool { return strings.HasPrefix(h.os, "Windows") } + // linuxHosts is every host whose agent tails journald or a file -- i.e. // everything that also produces the `system` service stream. Windows -// hosts produce eventlog records instead, and the stale host produces -// nothing at all. +// hosts produce their own channel records instead, and the stale host +// produces nothing at all. func linuxHosts() []*host { var out []*host for i := range fleet { h := &fleet[i] - if h.stale || h.service == "eventlog" { + if h.stale || h.windows() { continue } out = append(out, h)