Monitoring: alerts over metrics, by event and by email (MON-25 to MON-30)
Every minute the node that calculates metrics evaluates each enabled
x:Alert, read from the registry, with metric('name') or the underscore
name. An alert fires when its condition turns true: it emits
telemetry.alert-event with the rendered message, and queues one email per
recipient through the outbound queue, placeholders filled and
Auto-Submitted set. An alert naming an unknown metric is refused on save.
The shared alerts suite runs; every telemetry suite is un-gated.
This commit is contained in:
@@ -606,7 +606,23 @@ impl RegistrySet for Server {
|
||||
}
|
||||
|
||||
// Validate expressions
|
||||
if let Some(expressions) = new_object.inner.expression_ctxs() {
|
||||
// inbuxa: MON-25: an alert condition may name metrics with underscores
|
||||
let alert_condition = match &new_object.inner {
|
||||
ObjectInner::Alert(alert) => Some(
|
||||
common::telemetry::alerts::rewrite_condition(&alert.condition),
|
||||
),
|
||||
_ => None,
|
||||
};
|
||||
let expressions = match (&new_object.inner, &alert_condition) {
|
||||
(ObjectInner::Alert(alert), Some(condition)) => {
|
||||
Some(vec![registry::schema::prelude::ExpressionContext {
|
||||
expr: condition,
|
||||
..alert.ctx_condition()
|
||||
}])
|
||||
}
|
||||
_ => new_object.inner.expression_ctxs(),
|
||||
};
|
||||
if let Some(expressions) = expressions {
|
||||
let mut bp = Bootstrap::new_uninitialized(self.registry().clone());
|
||||
|
||||
for expression in expressions {
|
||||
|
||||
Reference in New Issue
Block a user