From 7fabc6a067466f1df3120457b923fdd93f3219f3 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 4 Sep 2026 20:19:51 -0700 Subject: [PATCH] Build the Phase 8 conformance corpus The design argues the conformance suite is the specification and should be built before either implementation, since two hand-written implementations of one language diverge unless something shared pins them. This is that suite: 38 cases in /processing, a language-neutral top-level directory for the same reason /proto is one -- the Rust agent and the Go ingest tier both consume the definition and neither owns it. Nothing executes the cases, because neither implementation exists. A stdlib-only validator checks the corpus stays well-formed and runs in CI: known actions, addressable fields, compilable patterns, names matching filenames, and no case depending on record_id, which is withheld so the question of whether an ingest-side rule can see one stays open. The validator was checked against seven deliberately broken cases before being trusted, since "38/38 valid" means nothing from a validator that cannot fail. Writing the cases first has already paid for itself twice. It forced two determinism decisions the prose had left vague, both of which a conformance suite cannot avoid answering. Sampling is counter-based rather than random: random is statistically nicer and impossible to assert on. Windows are measured on record timestamps rather than wall-clock, which makes replay deterministic and, not incidentally, makes backfill behave correctly where a wall-clock window would not. And it made the missing aggregate_count answer concrete. The design does not say what that action emits, or what a query not expecting a synthetic record sees. Rather than invent one by writing cases, the validator rejects any case using it, so the design question has to be answered before the behaviour can be frozen by accident. The corpus includes the acceptance case from real measured data: the two processes that account for roughly 60% of a real workstation's journal volume, and the one kernel message worth keeping. Signed-off-by: John Coffey --- .github/workflows/processing-conformance.yml | 29 +++ README.md | 1 + docs/phase-8-processing-design.md | 15 +- processing/README.md | 150 ++++++++++++ .../cases/actions_apply_in_order.json | 43 ++++ .../derive_copies_from_another_field.json | 38 +++ .../derive_overwrites_an_existing_value.json | 40 ++++ .../cases/derive_sets_a_literal_value.json | 38 +++ ...s_on_absent_attribute_is_not_an_error.json | 41 ++++ ..._fields_removes_only_named_attributes.json | 44 ++++ ..._stops_later_actions_in_the_same_rule.json | 30 +++ .../cases/drop_stops_later_rules.json | 60 +++++ .../empty_rule_set_is_a_pass_through.json | 45 ++++ ...keep_fields_is_an_attribute_allowlist.json | 43 ++++ ...leaves_non_matching_records_untouched.json | 37 +++ .../cases/mask_replaces_every_occurrence.json | 37 +++ ..._absent_attribute_is_not_empty_string.json | 50 ++++ .../cases/match_all_clauses_must_hold.json | 69 ++++++ .../cases/match_contains_prefix_suffix.json | 112 +++++++++ ...atch_empty_matcher_matches_everything.json | 33 +++ .../match_eq_is_exact_not_substring.json | 48 ++++ .../cases/match_exists_on_attribute.json | 62 +++++ .../match_ne_matches_when_different.json | 48 ++++ .../cases/match_on_severity_by_enum_name.json | 48 ++++ .../cases/match_regex_is_unanchored.json | 56 +++++ ...hing_rule_leaves_the_record_untouched.json | 47 ++++ ...on_lifts_top_level_keys_to_attributes.json | 38 +++ ...parse_json_on_invalid_json_is_a_no_op.json | 35 +++ .../parse_json_prefixes_keys_when_asked.json | 41 ++++ ...egex_named_captures_become_attributes.json | 39 ++++ ..._regex_that_does_not_match_is_a_no_op.json | 36 +++ .../rename_from_absent_field_is_a_no_op.json | 40 ++++ .../cases/rename_moves_an_attribute.json | 40 ++++ ..._apply_in_sequence_to_the_same_record.json | 55 +++++ .../sample_counts_only_matching_records.json | 81 +++++++ ...sample_keeps_the_first_then_every_nth.json | 99 ++++++++ .../cases/sample_of_one_keeps_everything.json | 51 ++++ ...uplicates_collapses_within_the_window.json | 51 ++++ ...plicates_emits_again_after_the_window.json | 51 ++++ ...uppress_duplicates_honours_key_fields.json | 63 +++++ ...duplicates_keys_on_message_by_default.json | 59 +++++ .../cases/workstation_noise_reduction.json | 103 +++++++++ processing/conformance/validate.py | 217 ++++++++++++++++++ 43 files changed, 2360 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/processing-conformance.yml create mode 100644 processing/README.md create mode 100644 processing/conformance/cases/actions_apply_in_order.json create mode 100644 processing/conformance/cases/derive_copies_from_another_field.json create mode 100644 processing/conformance/cases/derive_overwrites_an_existing_value.json create mode 100644 processing/conformance/cases/derive_sets_a_literal_value.json create mode 100644 processing/conformance/cases/drop_fields_on_absent_attribute_is_not_an_error.json create mode 100644 processing/conformance/cases/drop_fields_removes_only_named_attributes.json create mode 100644 processing/conformance/cases/drop_stops_later_actions_in_the_same_rule.json create mode 100644 processing/conformance/cases/drop_stops_later_rules.json create mode 100644 processing/conformance/cases/empty_rule_set_is_a_pass_through.json create mode 100644 processing/conformance/cases/keep_fields_is_an_attribute_allowlist.json create mode 100644 processing/conformance/cases/mask_leaves_non_matching_records_untouched.json create mode 100644 processing/conformance/cases/mask_replaces_every_occurrence.json create mode 100644 processing/conformance/cases/match_absent_attribute_is_not_empty_string.json create mode 100644 processing/conformance/cases/match_all_clauses_must_hold.json create mode 100644 processing/conformance/cases/match_contains_prefix_suffix.json create mode 100644 processing/conformance/cases/match_empty_matcher_matches_everything.json create mode 100644 processing/conformance/cases/match_eq_is_exact_not_substring.json create mode 100644 processing/conformance/cases/match_exists_on_attribute.json create mode 100644 processing/conformance/cases/match_ne_matches_when_different.json create mode 100644 processing/conformance/cases/match_on_severity_by_enum_name.json create mode 100644 processing/conformance/cases/match_regex_is_unanchored.json create mode 100644 processing/conformance/cases/non_matching_rule_leaves_the_record_untouched.json create mode 100644 processing/conformance/cases/parse_json_lifts_top_level_keys_to_attributes.json create mode 100644 processing/conformance/cases/parse_json_on_invalid_json_is_a_no_op.json create mode 100644 processing/conformance/cases/parse_json_prefixes_keys_when_asked.json create mode 100644 processing/conformance/cases/parse_regex_named_captures_become_attributes.json create mode 100644 processing/conformance/cases/parse_regex_that_does_not_match_is_a_no_op.json create mode 100644 processing/conformance/cases/rename_from_absent_field_is_a_no_op.json create mode 100644 processing/conformance/cases/rename_moves_an_attribute.json create mode 100644 processing/conformance/cases/rules_apply_in_sequence_to_the_same_record.json create mode 100644 processing/conformance/cases/sample_counts_only_matching_records.json create mode 100644 processing/conformance/cases/sample_keeps_the_first_then_every_nth.json create mode 100644 processing/conformance/cases/sample_of_one_keeps_everything.json create mode 100644 processing/conformance/cases/suppress_duplicates_collapses_within_the_window.json create mode 100644 processing/conformance/cases/suppress_duplicates_emits_again_after_the_window.json create mode 100644 processing/conformance/cases/suppress_duplicates_honours_key_fields.json create mode 100644 processing/conformance/cases/suppress_duplicates_keys_on_message_by_default.json create mode 100644 processing/conformance/cases/workstation_noise_reduction.json create mode 100755 processing/conformance/validate.py diff --git a/.github/workflows/processing-conformance.yml b/.github/workflows/processing-conformance.yml new file mode 100644 index 0000000..55cbc92 --- /dev/null +++ b/.github/workflows/processing-conformance.yml @@ -0,0 +1,29 @@ +name: Processing conformance corpus + +# /processing/conformance/cases IS the Phase 8 rule language spec: two +# implementations (Rust in the agent, Go at ingest) will each be held to +# it, and two hand-written implementations of one language diverge +# unless something shared pins them. +# +# Neither implementation exists yet, so nothing here executes rules. +# What this checks is that the corpus stays well-formed in the meantime +# -- known actions, addressable fields, compilable patterns, no case +# quietly depending on record_id. Cheap insurance against the spec +# rotting during the gap between writing it and building against it. +# +# Its own workflow, following web-routes.yml's reasoning: license- +# compliance.yml already carries one unrelated check for historical +# reasons, and that is not worth compounding. + +on: + push: + branches: [master, main] + pull_request: + +jobs: + conformance-corpus: + name: processing conformance corpus check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: python3 processing/conformance/validate.py diff --git a/README.md b/README.md index 8d79bd9..ae5424b 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ metadata/ PostgreSQL schema and migrations web/ SvelteKit frontend cli/ cairnobsctl proto/ gRPC service definitions +processing/ Phase 8 rule language spec and conformance corpus (no implementation yet) terraform/ Terraform provider enterprise/ SSO, multi-tenancy, per-tenant provisioning (off the roadmap) deploy/ Helm charts and Kubernetes operator diff --git a/docs/phase-8-processing-design.md b/docs/phase-8-processing-design.md index b7af84f..05e60e2 100644 --- a/docs/phase-8-processing-design.md +++ b/docs/phase-8-processing-design.md @@ -153,15 +153,24 @@ the same thing in both places" is the whole promise, and two hand-written implementations will diverge — not maybe, eventually. The deliverable that prevents it is a **language-neutral conformance -suite**: a directory of cases, each an input record, a rule set, and the -expected output record, in JSON. Both implementations run it in their own -CI. A case is added for every bug found in either. +suite**: a directory of cases, each a rule set, a sequence of input +records, and the records expected out, in JSON. Both implementations run +it in their own CI. A case is added for every bug found in either. This is the same discipline `/hack`'s fixtures already apply to ingest shapes, applied to semantics instead. It should be built **first**, not last — the suite is the specification, and the prose above is a summary of it. +**Built:** [`/processing`](../processing/README.md), 38 cases. Nothing +executes them yet, since neither implementation exists; a structural +validator runs in CI so the corpus cannot rot in the meantime. Writing +the cases first has already paid for itself — it forced two determinism +decisions that prose had left vague (see that README's "Two determinism +decisions the suite forces"), and it made the absence of an +`aggregate_count` answer concrete enough that the validator rejects any +case using it. + ## Decision 4: distribution reuses the channel that exists Fleet management already delivers desired state diff --git a/processing/README.md b/processing/README.md new file mode 100644 index 0000000..6c5a550 --- /dev/null +++ b/processing/README.md @@ -0,0 +1,150 @@ +# processing + +The Phase 8 processing rule language: its specification, and the +conformance suite that *is* that specification. + +> **Status:** the suite exists; the implementations do not. Nothing in +> Phase 8 is built. See +> [`/docs/phase-8-processing-design.md`](../docs/phase-8-processing-design.md) +> for the design and its open questions. + +## Why this is a top-level directory + +Rules run in two places, in two languages: the Rust agent +(`/agent`) and the Go ingest tier (`/ingest`). Neither owns the +definition. This is the same shape as `/proto` — a language-neutral +contract that several modules consume and none of them is the source of +truth for. + +## Why the cases are the specification + +Two hand-written implementations of one language will diverge. Not +maybe, eventually. The only thing that reliably prevents it is a shared +corpus both must satisfy, written down before either exists. + +So the prose below is a summary of `conformance/cases/`, not the other +way round. When they disagree, the cases win, and the prose is the bug. + +## Running it + +Nothing executes the cases yet, because neither implementation exists. +What runs today is a structural check that every case is well-formed — +valid JSON, known actions, no unknown keys, declared fields that exist: + +```sh +python3 processing/conformance/validate.py +``` + +That is worth having on its own: it stops the corpus rotting between now +and the first implementation, and it fails loudly if someone adds a case +using an action the spec does not define. + +Each implementation is expected to add a test that walks +`conformance/cases/*.json`, feeds `inputs` through `rules`, and asserts +the emitted records equal `expect`. Those runners are part of building +each side, not part of this directory. + +## Case format + +One JSON object per file: + +```json +{ + "name": "drop_discards_the_record", + "description": "A matched drop emits nothing at all.", + "rules": [ + { + "match": [{"field": "message", "op": "eq", "value": "noise"}], + "actions": [{"action": "drop"}] + } + ], + "inputs": [ + {"timestamp_unix_nano": 1000, "host": "h1", "service": "s1", + "severity": "SEVERITY_INFO", "message": "noise", "attributes": {}} + ], + "expect": [] +} +``` + +`inputs` is always a list, even for a single record, because windowed +actions need a sequence. `expect` is the records emitted, in order. + +### Records + +Mirrors `LogRecord` in `/proto/sentry/logs/v1/logs.proto`, snake_case, +with `severity` as the enum's name string. + +`record_id` is deliberately absent from every case. The agent never sets +it, and whether an ingest-side rule can see one depends on where in the +ingest path rules run — an open question. No case may depend on it, so +that decision stays free. + +### Matching + +A rule matches when **every** clause in `match` holds. An empty `match` +matches every record. + +| `op` | Meaning | +|---|---| +| `eq` / `ne` | exact string equality | +| `contains` | substring | +| `prefix` / `suffix` | string boundary | +| `regex` | linear-time regex, unanchored | +| `exists` / `not_exists` | field present (no `value`) | + +Addressable fields: `message`, `host`, `service`, `severity`, and +`attributes.`. A field that does not exist compares as absent, not +as empty string — `eq ""` does not match a missing attribute. + +### Actions + +Applied in order. A `drop` ends processing for that record immediately; +no later action or rule runs. + +| Action | Parameters | +|---|---| +| `drop` | — | +| `drop_fields` | `fields` | +| `keep_fields` | `fields` (attributes only; never removes top-level fields) | +| `rename` | `from`, `to` | +| `derive` | `field`, and one of `value` / `from_field` | +| `mask` | `field`, `pattern`, `replacement` | +| `parse_json` | `field` (default `message`), optional `prefix` | +| `parse_regex` | `field`, `pattern` (named captures become attributes) | +| `sample` | `keep_one_in` | +| `suppress_duplicates` | `window_ms`, optional `key_fields` | + +Rules are evaluated in the order given. Every matching rule's actions +apply, to the record as left by the rule before it. + +### Two determinism decisions the suite forces + +Conformance testing cannot assert on nondeterminism, so two things that +are usually left vague are pinned here: + +**`sample` is counter-based, not random.** `keep_one_in: 3` keeps the +first record and every third thereafter, per rule, per process. Random +sampling is statistically nicer and untestable; a counter is testable and +close enough at volume. + +**Windows are measured on record timestamps, not wall-clock.** A +`suppress_duplicates` window of 5000ms compares +`timestamp_unix_nano` values, so replaying the same input always gives +the same output regardless of how fast the test runs. This also means the +behaviour is correct under backfill, which wall-clock windows are not. + +### `aggregate_count` has no cases, deliberately + +The design lists it as an action but does not answer what it emits, or +what a query that is not expecting a synthetic record sees. Writing cases +now would invent that answer by accident and freeze it. It stays +unspecified until that question is decided — +[`phase-8-processing-design.md`](../docs/phase-8-processing-design.md) +open question 5. + +## Adding a case + +One behaviour per file, named for the behaviour rather than the action. +A case is added for every bug found in either implementation — that is +the mechanism that keeps the two from drifting, and it only works if the +case lands with the fix. diff --git a/processing/conformance/cases/actions_apply_in_order.json b/processing/conformance/cases/actions_apply_in_order.json new file mode 100644 index 0000000..19f4afc --- /dev/null +++ b/processing/conformance/cases/actions_apply_in_order.json @@ -0,0 +1,43 @@ +{ + "name": "actions_apply_in_order", + "description": "The second action sees what the first produced: parse then drop one of the parsed fields.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "parse_json", + "field": "message" + }, + { + "action": "drop_fields", + "fields": [ + "attributes.secret" + ] + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "{\"keep\":\"k\",\"secret\":\"s\"}", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "{\"keep\":\"k\",\"secret\":\"s\"}", + "attributes": { + "keep": "k" + } + } + ] +} diff --git a/processing/conformance/cases/derive_copies_from_another_field.json b/processing/conformance/cases/derive_copies_from_another_field.json new file mode 100644 index 0000000..48f1c75 --- /dev/null +++ b/processing/conformance/cases/derive_copies_from_another_field.json @@ -0,0 +1,38 @@ +{ + "name": "derive_copies_from_another_field", + "description": "derive with from_field copies, leaving the source in place.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "derive", + "field": "attributes.origin_host", + "from_field": "host" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "web-01", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "web-01", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "origin_host": "web-01" + } + } + ] +} diff --git a/processing/conformance/cases/derive_overwrites_an_existing_value.json b/processing/conformance/cases/derive_overwrites_an_existing_value.json new file mode 100644 index 0000000..d270a42 --- /dev/null +++ b/processing/conformance/cases/derive_overwrites_an_existing_value.json @@ -0,0 +1,40 @@ +{ + "name": "derive_overwrites_an_existing_value", + "description": "derive is a set, not an insert-if-absent.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "derive", + "field": "attributes.env", + "value": "dev" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "env": "prod" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "env": "dev" + } + } + ] +} diff --git a/processing/conformance/cases/derive_sets_a_literal_value.json b/processing/conformance/cases/derive_sets_a_literal_value.json new file mode 100644 index 0000000..cc2fc6a --- /dev/null +++ b/processing/conformance/cases/derive_sets_a_literal_value.json @@ -0,0 +1,38 @@ +{ + "name": "derive_sets_a_literal_value", + "description": "derive with value writes a constant.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "derive", + "field": "attributes.env", + "value": "dev" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "env": "dev" + } + } + ] +} diff --git a/processing/conformance/cases/drop_fields_on_absent_attribute_is_not_an_error.json b/processing/conformance/cases/drop_fields_on_absent_attribute_is_not_an_error.json new file mode 100644 index 0000000..d4af328 --- /dev/null +++ b/processing/conformance/cases/drop_fields_on_absent_attribute_is_not_an_error.json @@ -0,0 +1,41 @@ +{ + "name": "drop_fields_on_absent_attribute_is_not_an_error", + "description": "Removing something that is not there succeeds and changes nothing.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "drop_fields", + "fields": [ + "attributes.nope" + ] + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "keep": "k" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "keep": "k" + } + } + ] +} diff --git a/processing/conformance/cases/drop_fields_removes_only_named_attributes.json b/processing/conformance/cases/drop_fields_removes_only_named_attributes.json new file mode 100644 index 0000000..dbe41b0 --- /dev/null +++ b/processing/conformance/cases/drop_fields_removes_only_named_attributes.json @@ -0,0 +1,44 @@ +{ + "name": "drop_fields_removes_only_named_attributes", + "description": "drop_fields removes the named attributes and leaves the rest alone.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "drop_fields", + "fields": [ + "attributes.secret", + "attributes.gone" + ] + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "secret": "s", + "gone": "g", + "keep": "k" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "keep": "k" + } + } + ] +} diff --git a/processing/conformance/cases/drop_stops_later_actions_in_the_same_rule.json b/processing/conformance/cases/drop_stops_later_actions_in_the_same_rule.json new file mode 100644 index 0000000..33eab2b --- /dev/null +++ b/processing/conformance/cases/drop_stops_later_actions_in_the_same_rule.json @@ -0,0 +1,30 @@ +{ + "name": "drop_stops_later_actions_in_the_same_rule", + "description": "Nothing after a drop runs; the record is gone, so a subsequent derive cannot resurrect it.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "drop" + }, + { + "action": "derive", + "field": "attributes.late", + "value": "x" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + } + ], + "expect": [] +} diff --git a/processing/conformance/cases/drop_stops_later_rules.json b/processing/conformance/cases/drop_stops_later_rules.json new file mode 100644 index 0000000..c6380fe --- /dev/null +++ b/processing/conformance/cases/drop_stops_later_rules.json @@ -0,0 +1,60 @@ +{ + "name": "drop_stops_later_rules", + "description": "A drop in rule 1 means rule 2 never sees the record.", + "rules": [ + { + "match": [ + { + "field": "message", + "op": "eq", + "value": "gone" + } + ], + "actions": [ + { + "action": "drop" + } + ] + }, + { + "match": [], + "actions": [ + { + "action": "derive", + "field": "attributes.seen", + "value": "yes" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "gone", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "stays", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "stays", + "attributes": { + "seen": "yes" + } + } + ] +} diff --git a/processing/conformance/cases/empty_rule_set_is_a_pass_through.json b/processing/conformance/cases/empty_rule_set_is_a_pass_through.json new file mode 100644 index 0000000..9f995fd --- /dev/null +++ b/processing/conformance/cases/empty_rule_set_is_a_pass_through.json @@ -0,0 +1,45 @@ +{ + "name": "empty_rule_set_is_a_pass_through", + "description": "No rules means every record is emitted exactly as it arrived. This is the state an agent falls back to when it refuses a rule set.", + "rules": [], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "k": "v" + } + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "k": "v" + } + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/keep_fields_is_an_attribute_allowlist.json b/processing/conformance/cases/keep_fields_is_an_attribute_allowlist.json new file mode 100644 index 0000000..e8afb27 --- /dev/null +++ b/processing/conformance/cases/keep_fields_is_an_attribute_allowlist.json @@ -0,0 +1,43 @@ +{ + "name": "keep_fields_is_an_attribute_allowlist", + "description": "keep_fields drops every attribute not named, and never touches top-level fields.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "keep_fields", + "fields": [ + "attributes.keep" + ] + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h9", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "message survives", + "attributes": { + "keep": "k", + "a": "1", + "b": "2" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h9", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "message survives", + "attributes": { + "keep": "k" + } + } + ] +} diff --git a/processing/conformance/cases/mask_leaves_non_matching_records_untouched.json b/processing/conformance/cases/mask_leaves_non_matching_records_untouched.json new file mode 100644 index 0000000..48a3bb8 --- /dev/null +++ b/processing/conformance/cases/mask_leaves_non_matching_records_untouched.json @@ -0,0 +1,37 @@ +{ + "name": "mask_leaves_non_matching_records_untouched", + "description": "A mask whose pattern does not hit changes nothing at all.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "mask", + "field": "message", + "pattern": "[0-9]{16}", + "replacement": "[REDACTED]" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "nothing sensitive here", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "nothing sensitive here", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/mask_replaces_every_occurrence.json b/processing/conformance/cases/mask_replaces_every_occurrence.json new file mode 100644 index 0000000..6c8c222 --- /dev/null +++ b/processing/conformance/cases/mask_replaces_every_occurrence.json @@ -0,0 +1,37 @@ +{ + "name": "mask_replaces_every_occurrence", + "description": "mask replaces all matches in the field, not just the first -- redaction that stops at the first hit is a leak.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "mask", + "field": "message", + "pattern": "[0-9]{16}", + "replacement": "[REDACTED]" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "card 4111111111111111 and 5500000000000004 seen", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "card [REDACTED] and [REDACTED] seen", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_absent_attribute_is_not_empty_string.json b/processing/conformance/cases/match_absent_attribute_is_not_empty_string.json new file mode 100644 index 0000000..4cb0449 --- /dev/null +++ b/processing/conformance/cases/match_absent_attribute_is_not_empty_string.json @@ -0,0 +1,50 @@ +{ + "name": "match_absent_attribute_is_not_empty_string", + "description": "A missing attribute must not compare equal to \"\" -- absence and emptiness are different.", + "rules": [ + { + "match": [ + { + "field": "attributes.tier", + "op": "eq", + "value": "" + } + ], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "missing", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "empty", + "attributes": { + "tier": "" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "missing", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_all_clauses_must_hold.json b/processing/conformance/cases/match_all_clauses_must_hold.json new file mode 100644 index 0000000..9933144 --- /dev/null +++ b/processing/conformance/cases/match_all_clauses_must_hold.json @@ -0,0 +1,69 @@ +{ + "name": "match_all_clauses_must_hold", + "description": "Multiple clauses are AND, never OR.", + "rules": [ + { + "match": [ + { + "field": "host", + "op": "eq", + "value": "h1" + }, + { + "field": "severity", + "op": "eq", + "value": "SEVERITY_ERROR" + } + ], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_ERROR", + "message": "both", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "host only", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000, + "host": "h2", + "service": "s1", + "severity": "SEVERITY_ERROR", + "message": "sev only", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "host only", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000, + "host": "h2", + "service": "s1", + "severity": "SEVERITY_ERROR", + "message": "sev only", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_contains_prefix_suffix.json b/processing/conformance/cases/match_contains_prefix_suffix.json new file mode 100644 index 0000000..62770a2 --- /dev/null +++ b/processing/conformance/cases/match_contains_prefix_suffix.json @@ -0,0 +1,112 @@ +{ + "name": "match_contains_prefix_suffix", + "description": "The three substring operators, each matching exactly one of three records.", + "rules": [ + { + "match": [ + { + "field": "message", + "op": "prefix", + "value": "start" + } + ], + "actions": [ + { + "action": "derive", + "field": "attributes.hit", + "value": "prefix" + } + ] + }, + { + "match": [ + { + "field": "message", + "op": "suffix", + "value": "end" + } + ], + "actions": [ + { + "action": "derive", + "field": "attributes.hit", + "value": "suffix" + } + ] + }, + { + "match": [ + { + "field": "message", + "op": "contains", + "value": "middle" + } + ], + "actions": [ + { + "action": "derive", + "field": "attributes.hit", + "value": "contains" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "start of line", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "line to end", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a middle b", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "start of line", + "attributes": { + "hit": "prefix" + } + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "line to end", + "attributes": { + "hit": "suffix" + } + }, + { + "timestamp_unix_nano": 3000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a middle b", + "attributes": { + "hit": "contains" + } + } + ] +} diff --git a/processing/conformance/cases/match_empty_matcher_matches_everything.json b/processing/conformance/cases/match_empty_matcher_matches_everything.json new file mode 100644 index 0000000..d45c461 --- /dev/null +++ b/processing/conformance/cases/match_empty_matcher_matches_everything.json @@ -0,0 +1,33 @@ +{ + "name": "match_empty_matcher_matches_everything", + "description": "An empty match list is 'always', not 'never'.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "anything", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "anything else", + "attributes": {} + } + ], + "expect": [] +} diff --git a/processing/conformance/cases/match_eq_is_exact_not_substring.json b/processing/conformance/cases/match_eq_is_exact_not_substring.json new file mode 100644 index 0000000..a9b6365 --- /dev/null +++ b/processing/conformance/cases/match_eq_is_exact_not_substring.json @@ -0,0 +1,48 @@ +{ + "name": "match_eq_is_exact_not_substring", + "description": "eq compares the whole value; a superstring does not match.", + "rules": [ + { + "match": [ + { + "field": "message", + "op": "eq", + "value": "noise" + } + ], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "noise", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "noise and more", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "noise and more", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_exists_on_attribute.json b/processing/conformance/cases/match_exists_on_attribute.json new file mode 100644 index 0000000..d3240c6 --- /dev/null +++ b/processing/conformance/cases/match_exists_on_attribute.json @@ -0,0 +1,62 @@ +{ + "name": "match_exists_on_attribute", + "description": "exists/not_exists test presence and take no value.", + "rules": [ + { + "match": [ + { + "field": "attributes.trace_id", + "op": "exists" + } + ], + "actions": [ + { + "action": "derive", + "field": "attributes.traced", + "value": "yes" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "trace_id": "abc" + } + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "trace_id": "abc", + "traced": "yes" + } + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_ne_matches_when_different.json b/processing/conformance/cases/match_ne_matches_when_different.json new file mode 100644 index 0000000..c441d2d --- /dev/null +++ b/processing/conformance/cases/match_ne_matches_when_different.json @@ -0,0 +1,48 @@ +{ + "name": "match_ne_matches_when_different", + "description": "ne is the complement of eq, including for absent fields.", + "rules": [ + { + "match": [ + { + "field": "service", + "op": "ne", + "value": "keep-me" + } + ], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "keep-me", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "other", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "keep-me", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_on_severity_by_enum_name.json b/processing/conformance/cases/match_on_severity_by_enum_name.json new file mode 100644 index 0000000..ce4a500 --- /dev/null +++ b/processing/conformance/cases/match_on_severity_by_enum_name.json @@ -0,0 +1,48 @@ +{ + "name": "match_on_severity_by_enum_name", + "description": "severity is matched by its enum name string, not its number.", + "rules": [ + { + "match": [ + { + "field": "severity", + "op": "eq", + "value": "SEVERITY_DEBUG" + } + ], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_DEBUG", + "message": "debug", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "info", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "info", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/match_regex_is_unanchored.json b/processing/conformance/cases/match_regex_is_unanchored.json new file mode 100644 index 0000000..7f3e74c --- /dev/null +++ b/processing/conformance/cases/match_regex_is_unanchored.json @@ -0,0 +1,56 @@ +{ + "name": "match_regex_is_unanchored", + "description": "regex matches anywhere in the value unless the pattern anchors itself.", + "rules": [ + { + "match": [ + { + "field": "message", + "op": "regex", + "value": "conn(ect|ected)" + } + ], + "actions": [ + { + "action": "drop" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "client connected ok", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "disconnected", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "unrelated", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 3000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "unrelated", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/non_matching_rule_leaves_the_record_untouched.json b/processing/conformance/cases/non_matching_rule_leaves_the_record_untouched.json new file mode 100644 index 0000000..687923d --- /dev/null +++ b/processing/conformance/cases/non_matching_rule_leaves_the_record_untouched.json @@ -0,0 +1,47 @@ +{ + "name": "non_matching_rule_leaves_the_record_untouched", + "description": "A rule that does not match is not a rule that empties things.", + "rules": [ + { + "match": [ + { + "field": "host", + "op": "eq", + "value": "other" + } + ], + "actions": [ + { + "action": "drop_fields", + "fields": [ + "attributes.keep" + ] + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "keep": "k" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "keep": "k" + } + } + ] +} diff --git a/processing/conformance/cases/parse_json_lifts_top_level_keys_to_attributes.json b/processing/conformance/cases/parse_json_lifts_top_level_keys_to_attributes.json new file mode 100644 index 0000000..798962e --- /dev/null +++ b/processing/conformance/cases/parse_json_lifts_top_level_keys_to_attributes.json @@ -0,0 +1,38 @@ +{ + "name": "parse_json_lifts_top_level_keys_to_attributes", + "description": "parse_json on message adds string attributes and leaves message intact.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "parse_json", + "field": "message" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "{\"level\":\"warn\",\"code\":\"503\"}", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "{\"level\":\"warn\",\"code\":\"503\"}", + "attributes": { + "level": "warn", + "code": "503" + } + } + ] +} diff --git a/processing/conformance/cases/parse_json_on_invalid_json_is_a_no_op.json b/processing/conformance/cases/parse_json_on_invalid_json_is_a_no_op.json new file mode 100644 index 0000000..0871812 --- /dev/null +++ b/processing/conformance/cases/parse_json_on_invalid_json_is_a_no_op.json @@ -0,0 +1,35 @@ +{ + "name": "parse_json_on_invalid_json_is_a_no_op", + "description": "A record that is not JSON passes through unchanged rather than being dropped or erroring -- same schema-on-read fallback the agent parser already uses.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "parse_json", + "field": "message" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "this is not json {", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "this is not json {", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/parse_json_prefixes_keys_when_asked.json b/processing/conformance/cases/parse_json_prefixes_keys_when_asked.json new file mode 100644 index 0000000..25af227 --- /dev/null +++ b/processing/conformance/cases/parse_json_prefixes_keys_when_asked.json @@ -0,0 +1,41 @@ +{ + "name": "parse_json_prefixes_keys_when_asked", + "description": "prefix namespaces the lifted keys so they cannot collide with existing attributes.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "parse_json", + "field": "message", + "prefix": "json." + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "{\"code\":\"503\"}", + "attributes": { + "code": "keep-me" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "{\"code\":\"503\"}", + "attributes": { + "code": "keep-me", + "json.code": "503" + } + } + ] +} diff --git a/processing/conformance/cases/parse_regex_named_captures_become_attributes.json b/processing/conformance/cases/parse_regex_named_captures_become_attributes.json new file mode 100644 index 0000000..71bf229 --- /dev/null +++ b/processing/conformance/cases/parse_regex_named_captures_become_attributes.json @@ -0,0 +1,39 @@ +{ + "name": "parse_regex_named_captures_become_attributes", + "description": "Only named groups are lifted; unnamed groups are ignored.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "parse_regex", + "field": "message", + "pattern": "user=(?P[a-z]+) code=(?P[0-9]+)" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "user=alice code=403 extra", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "user=alice code=403 extra", + "attributes": { + "user": "alice", + "code": "403" + } + } + ] +} diff --git a/processing/conformance/cases/parse_regex_that_does_not_match_is_a_no_op.json b/processing/conformance/cases/parse_regex_that_does_not_match_is_a_no_op.json new file mode 100644 index 0000000..d03f32f --- /dev/null +++ b/processing/conformance/cases/parse_regex_that_does_not_match_is_a_no_op.json @@ -0,0 +1,36 @@ +{ + "name": "parse_regex_that_does_not_match_is_a_no_op", + "description": "A non-matching pattern adds nothing and does not drop the record.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "parse_regex", + "field": "message", + "pattern": "user=(?P[a-z]+)" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "no user here", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "no user here", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/rename_from_absent_field_is_a_no_op.json b/processing/conformance/cases/rename_from_absent_field_is_a_no_op.json new file mode 100644 index 0000000..3285d60 --- /dev/null +++ b/processing/conformance/cases/rename_from_absent_field_is_a_no_op.json @@ -0,0 +1,40 @@ +{ + "name": "rename_from_absent_field_is_a_no_op", + "description": "Renaming something absent does not create an empty destination.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "rename", + "from": "attributes.old", + "to": "attributes.new" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "other": "v" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "other": "v" + } + } + ] +} diff --git a/processing/conformance/cases/rename_moves_an_attribute.json b/processing/conformance/cases/rename_moves_an_attribute.json new file mode 100644 index 0000000..6b682af --- /dev/null +++ b/processing/conformance/cases/rename_moves_an_attribute.json @@ -0,0 +1,40 @@ +{ + "name": "rename_moves_an_attribute", + "description": "rename removes the source key and creates the destination.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "rename", + "from": "attributes.old", + "to": "attributes.new" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "old": "v" + } + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": { + "new": "v" + } + } + ] +} diff --git a/processing/conformance/cases/rules_apply_in_sequence_to_the_same_record.json b/processing/conformance/cases/rules_apply_in_sequence_to_the_same_record.json new file mode 100644 index 0000000..d2ada06 --- /dev/null +++ b/processing/conformance/cases/rules_apply_in_sequence_to_the_same_record.json @@ -0,0 +1,55 @@ +{ + "name": "rules_apply_in_sequence_to_the_same_record", + "description": "Every matching rule runs, each seeing the record as the previous rule left it.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "derive", + "field": "attributes.a", + "value": "1" + } + ] + }, + { + "match": [ + { + "field": "attributes.a", + "op": "eq", + "value": "1" + } + ], + "actions": [ + { + "action": "derive", + "field": "attributes.b", + "value": "2" + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "x", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "x", + "attributes": { + "a": "1", + "b": "2" + } + } + ] +} diff --git a/processing/conformance/cases/sample_counts_only_matching_records.json b/processing/conformance/cases/sample_counts_only_matching_records.json new file mode 100644 index 0000000..f15c4a7 --- /dev/null +++ b/processing/conformance/cases/sample_counts_only_matching_records.json @@ -0,0 +1,81 @@ +{ + "name": "sample_counts_only_matching_records", + "description": "The counter advances on records the rule matched, so an unrelated record between two matches does not consume a slot.", + "rules": [ + { + "match": [ + { + "field": "service", + "op": "eq", + "value": "noisy" + } + ], + "actions": [ + { + "action": "sample", + "keep_one_in": 2 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "noisy", + "severity": "SEVERITY_INFO", + "message": "n1", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "quiet", + "severity": "SEVERITY_INFO", + "message": "other", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000000, + "host": "h1", + "service": "noisy", + "severity": "SEVERITY_INFO", + "message": "n2", + "attributes": {} + }, + { + "timestamp_unix_nano": 4000000, + "host": "h1", + "service": "noisy", + "severity": "SEVERITY_INFO", + "message": "n3", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "noisy", + "severity": "SEVERITY_INFO", + "message": "n1", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "quiet", + "severity": "SEVERITY_INFO", + "message": "other", + "attributes": {} + }, + { + "timestamp_unix_nano": 4000000, + "host": "h1", + "service": "noisy", + "severity": "SEVERITY_INFO", + "message": "n3", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/sample_keeps_the_first_then_every_nth.json b/processing/conformance/cases/sample_keeps_the_first_then_every_nth.json new file mode 100644 index 0000000..4b427d1 --- /dev/null +++ b/processing/conformance/cases/sample_keeps_the_first_then_every_nth.json @@ -0,0 +1,99 @@ +{ + "name": "sample_keeps_the_first_then_every_nth", + "description": "sample is counter-based, not random: keep_one_in 3 keeps records 1, 4, 7. Random sampling is untestable; a counter is testable and close enough at volume.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "sample", + "keep_one_in": 3 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m1", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m2", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m3", + "attributes": {} + }, + { + "timestamp_unix_nano": 4000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m4", + "attributes": {} + }, + { + "timestamp_unix_nano": 5000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m5", + "attributes": {} + }, + { + "timestamp_unix_nano": 6000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m6", + "attributes": {} + }, + { + "timestamp_unix_nano": 7000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m7", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m1", + "attributes": {} + }, + { + "timestamp_unix_nano": 4000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m4", + "attributes": {} + }, + { + "timestamp_unix_nano": 7000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "m7", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/sample_of_one_keeps_everything.json b/processing/conformance/cases/sample_of_one_keeps_everything.json new file mode 100644 index 0000000..4411bbe --- /dev/null +++ b/processing/conformance/cases/sample_of_one_keeps_everything.json @@ -0,0 +1,51 @@ +{ + "name": "sample_of_one_keeps_everything", + "description": "keep_one_in 1 is a no-op, not an off switch.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "sample", + "keep_one_in": 1 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/suppress_duplicates_collapses_within_the_window.json b/processing/conformance/cases/suppress_duplicates_collapses_within_the_window.json new file mode 100644 index 0000000..1ccf738 --- /dev/null +++ b/processing/conformance/cases/suppress_duplicates_collapses_within_the_window.json @@ -0,0 +1,51 @@ +{ + "name": "suppress_duplicates_collapses_within_the_window", + "description": "Identical messages inside the window collapse to the first; the window is measured on record timestamps, not wall-clock, so replay is deterministic and backfill behaves.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "suppress_duplicates", + "window_ms": 5000 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + }, + { + "timestamp_unix_nano": 1000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + }, + { + "timestamp_unix_nano": 4999000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/suppress_duplicates_emits_again_after_the_window.json b/processing/conformance/cases/suppress_duplicates_emits_again_after_the_window.json new file mode 100644 index 0000000..48c1709 --- /dev/null +++ b/processing/conformance/cases/suppress_duplicates_emits_again_after_the_window.json @@ -0,0 +1,51 @@ +{ + "name": "suppress_duplicates_emits_again_after_the_window", + "description": "A duplicate outside the window is a new first occurrence.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "suppress_duplicates", + "window_ms": 5000 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + }, + { + "timestamp_unix_nano": 5000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + }, + { + "timestamp_unix_nano": 5000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/suppress_duplicates_honours_key_fields.json b/processing/conformance/cases/suppress_duplicates_honours_key_fields.json new file mode 100644 index 0000000..2e7f07b --- /dev/null +++ b/processing/conformance/cases/suppress_duplicates_honours_key_fields.json @@ -0,0 +1,63 @@ +{ + "name": "suppress_duplicates_honours_key_fields", + "description": "With key_fields, identical messages from different hosts are not duplicates of each other.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "suppress_duplicates", + "window_ms": 5000, + "key_fields": [ + "host", + "message" + ] + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + }, + { + "timestamp_unix_nano": 1000000, + "host": "h2", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + }, + { + "timestamp_unix_nano": 1000000, + "host": "h2", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "same", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/suppress_duplicates_keys_on_message_by_default.json b/processing/conformance/cases/suppress_duplicates_keys_on_message_by_default.json new file mode 100644 index 0000000..b71dcec --- /dev/null +++ b/processing/conformance/cases/suppress_duplicates_keys_on_message_by_default.json @@ -0,0 +1,59 @@ +{ + "name": "suppress_duplicates_keys_on_message_by_default", + "description": "Two different messages never suppress each other.", + "rules": [ + { + "match": [], + "actions": [ + { + "action": "suppress_duplicates", + "window_ms": 5000 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + }, + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "a", + "attributes": {} + }, + { + "timestamp_unix_nano": 1000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "b", + "attributes": {} + } + ] +} diff --git a/processing/conformance/cases/workstation_noise_reduction.json b/processing/conformance/cases/workstation_noise_reduction.json new file mode 100644 index 0000000..1d9e084 --- /dev/null +++ b/processing/conformance/cases/workstation_noise_reduction.json @@ -0,0 +1,103 @@ +{ + "name": "workstation_noise_reduction", + "description": "The v1 acceptance shape, from real measured data: Discord repeating a byte-identical version string and a polling daemon repeating a fixed message, against one kernel message worth keeping. See /docs/phase-8-processing-design.md.", + "rules": [ + { + "match": [ + { + "field": "message", + "op": "eq", + "value": "Discord 1.0.155" + } + ], + "actions": [ + { + "action": "drop" + } + ] + }, + { + "match": [ + { + "field": "message", + "op": "eq", + "value": "Checking active tasks" + } + ], + "actions": [ + { + "action": "suppress_duplicates", + "window_ms": 60000 + } + ] + } + ], + "inputs": [ + { + "timestamp_unix_nano": 0, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + }, + { + "timestamp_unix_nano": 1000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Checking active tasks", + "attributes": {} + }, + { + "timestamp_unix_nano": 2000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + }, + { + "timestamp_unix_nano": 3000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Checking active tasks", + "attributes": {} + }, + { + "timestamp_unix_nano": 4000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_WARN", + "message": "[UFW BLOCK] IN=wlan0 SRC=172.16.16.1", + "attributes": {} + }, + { + "timestamp_unix_nano": 5000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Discord 1.0.155", + "attributes": {} + } + ], + "expect": [ + { + "timestamp_unix_nano": 1000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_INFO", + "message": "Checking active tasks", + "attributes": {} + }, + { + "timestamp_unix_nano": 4000000000, + "host": "h1", + "service": "s1", + "severity": "SEVERITY_WARN", + "message": "[UFW BLOCK] IN=wlan0 SRC=172.16.16.1", + "attributes": {} + } + ] +} diff --git a/processing/conformance/validate.py b/processing/conformance/validate.py new file mode 100755 index 0000000..a2195b2 --- /dev/null +++ b/processing/conformance/validate.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Structural check for the processing conformance corpus. + +The cases are the specification (see ../README.md), so they have to stay +well-formed even while neither implementation exists to execute them. +This validates shape only -- it does not run rules against records, +because there is nothing yet to run them with. + +Stdlib only, on purpose: this must be runnable anywhere without a +dependency install, including from a bare CI container. + +Exit 0 if every case is valid, 1 otherwise. +""" + +import json +import os +import re +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +CASES = os.path.join(HERE, "cases") + +CASE_KEYS = {"name", "description", "rules", "inputs", "expect"} +RECORD_KEYS = {"timestamp_unix_nano", "host", "service", "severity", + "message", "attributes"} + +SEVERITIES = {"SEVERITY_UNSPECIFIED", "SEVERITY_TRACE", "SEVERITY_DEBUG", + "SEVERITY_INFO", "SEVERITY_WARN", "SEVERITY_ERROR", + "SEVERITY_FATAL"} + +OPS_WITH_VALUE = {"eq", "ne", "contains", "prefix", "suffix", "regex"} +OPS_WITHOUT_VALUE = {"exists", "not_exists"} + +TOP_LEVEL_FIELDS = {"message", "host", "service", "severity"} + +# action -> (required params, optional params) +ACTIONS = { + "drop": (set(), set()), + "drop_fields": ({"fields"}, set()), + "keep_fields": ({"fields"}, set()), + "rename": ({"from", "to"}, set()), + "derive": ({"field"}, {"value", "from_field"}), + "mask": ({"field", "pattern", "replacement"}, set()), + "parse_json": ({"field"}, {"prefix"}), + "parse_regex": ({"field", "pattern"}, set()), + "sample": ({"keep_one_in"}, set()), + "suppress_duplicates": ({"window_ms"}, {"key_fields"}), +} + +# Deliberately absent from ACTIONS. The design does not say what it emits +# or what a query that is not expecting a synthetic record sees, so a +# case using it would invent that answer and freeze it by accident. +UNSPECIFIED_ACTIONS = {"aggregate_count"} + + +def valid_field(name): + return name in TOP_LEVEL_FIELDS or name.startswith("attributes.") + + +def check_record(rec, where, err): + if not isinstance(rec, dict): + return err(f"{where}: record must be an object") + missing = RECORD_KEYS - set(rec) + unknown = set(rec) - RECORD_KEYS + if missing: + err(f"{where}: record missing {sorted(missing)}") + if "record_id" in unknown: + err(f"{where}: record_id must never appear -- no case may depend " + f"on it (see ../README.md)") + if unknown - {"record_id"}: + err(f"{where}: record has unknown keys {sorted(unknown - {'record_id'})}") + if rec.get("severity") not in SEVERITIES: + err(f"{where}: unknown severity {rec.get('severity')!r}") + if not isinstance(rec.get("timestamp_unix_nano"), int): + err(f"{where}: timestamp_unix_nano must be an integer") + attrs = rec.get("attributes") + if not isinstance(attrs, dict): + err(f"{where}: attributes must be an object") + else: + for k, v in attrs.items(): + if not isinstance(k, str) or not isinstance(v, str): + err(f"{where}: attributes must be string->string, got {k!r}={v!r}") + + +def check_action(a, where, err): + if not isinstance(a, dict) or "action" not in a: + return err(f"{where}: action must be an object with an 'action' key") + name = a["action"] + if name in UNSPECIFIED_ACTIONS: + return err(f"{where}: '{name}' is deliberately unspecified -- decide " + f"its output shape in the design doc before writing cases " + f"for it") + if name not in ACTIONS: + return err(f"{where}: unknown action {name!r}") + required, optional = ACTIONS[name] + given = set(a) - {"action"} + if required - given: + err(f"{where}: action {name!r} missing {sorted(required - given)}") + if given - required - optional: + err(f"{where}: action {name!r} has unknown params " + f"{sorted(given - required - optional)}") + if name == "derive" and not ({"value", "from_field"} & given): + err(f"{where}: derive needs exactly one of value/from_field") + if name == "derive" and {"value", "from_field"} <= given: + err(f"{where}: derive needs exactly one of value/from_field, not both") + for key in ("field", "from", "to"): + if key in a and not valid_field(a[key]): + err(f"{where}: {key}={a[key]!r} is not an addressable field") + for key in ("fields", "key_fields"): + for f in a.get(key, []): + if not valid_field(f): + err(f"{where}: {key} entry {f!r} is not an addressable field") + for key in ("pattern",): + if key in a: + try: + re.compile(a[key]) + except re.error as exc: + err(f"{where}: {key} is not a valid regex: {exc}") + if name == "sample": + n = a.get("keep_one_in") + if not isinstance(n, int) or n < 1: + err(f"{where}: keep_one_in must be an integer >= 1") + if name == "suppress_duplicates": + w = a.get("window_ms") + if not isinstance(w, int) or w < 1: + err(f"{where}: window_ms must be an integer >= 1") + + +def check_case(path, err): + with open(path) as fh: + try: + case = json.load(fh) + except json.JSONDecodeError as exc: + return err(f"invalid JSON: {exc}") + + missing = CASE_KEYS - set(case) + unknown = set(case) - CASE_KEYS + if missing: + err(f"missing top-level keys {sorted(missing)}") + if unknown: + err(f"unknown top-level keys {sorted(unknown)}") + + expected_name = os.path.basename(path)[:-len(".json")] + if case.get("name") != expected_name: + err(f"name {case.get('name')!r} does not match filename " + f"{expected_name!r}") + if not case.get("description"): + err("description must not be empty -- it is what the case is for") + + for i, rule in enumerate(case.get("rules", [])): + where = f"rules[{i}]" + if not isinstance(rule, dict) or set(rule) != {"match", "actions"}: + err(f"{where}: a rule is exactly {{match, actions}}") + continue + for j, clause in enumerate(rule["match"]): + cw = f"{where}.match[{j}]" + if not isinstance(clause, dict): + err(f"{cw}: clause must be an object") + continue + op = clause.get("op") + if op in OPS_WITHOUT_VALUE: + if "value" in clause: + err(f"{cw}: {op} takes no value") + allowed = {"field", "op"} + elif op in OPS_WITH_VALUE: + if "value" not in clause: + err(f"{cw}: {op} requires a value") + allowed = {"field", "op", "value"} + else: + err(f"{cw}: unknown op {op!r}") + continue + if set(clause) - allowed: + err(f"{cw}: unknown keys {sorted(set(clause) - allowed)}") + if not valid_field(clause.get("field", "")): + err(f"{cw}: {clause.get('field')!r} is not an addressable field") + if op == "regex": + try: + re.compile(clause["value"]) + except re.error as exc: + err(f"{cw}: not a valid regex: {exc}") + if not rule["actions"]: + err(f"{where}: a rule with no actions does nothing") + for j, action in enumerate(rule["actions"]): + check_action(action, f"{where}.actions[{j}]", err) + + for i, rec in enumerate(case.get("inputs", [])): + check_record(rec, f"inputs[{i}]", err) + if not case.get("inputs"): + err("inputs must not be empty -- a case with no input asserts nothing") + for i, rec in enumerate(case.get("expect", [])): + check_record(rec, f"expect[{i}]", err) + + +def main(): + if not os.path.isdir(CASES): + print(f"no cases directory at {CASES}", file=sys.stderr) + return 1 + files = sorted(f for f in os.listdir(CASES) if f.endswith(".json")) + if not files: + print("no cases found", file=sys.stderr) + return 1 + + failures = 0 + for name in files: + errors = [] + check_case(os.path.join(CASES, name), errors.append) + if errors: + failures += 1 + print(f"FAIL {name}") + for e in errors: + print(f" {e}") + print(f"\n{len(files) - failures}/{len(files)} cases valid") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())