Monitoring: trace history, trace search, and x:Trace over the stored traces (MON-10 to MON-17, MON-32)
A lossy collector subscriber keeps each inbound SMTP session that reached MAIL FROM and each delivery attempt, info level and above and never raw I/O, at most 1,000 events with strings cut at 4 KiB, and writes it when the span closes as an x:Trace under the telemetry key class, scheduling its indexing. The index task builds a document of event types, queue ids and keywords when indexTelemetry is on. x:Trace/get derives timestamp, from, to and size; /query filters by opening event, text, queueId and time; /set destroys only. The data purge honours holdTracesFor. The shared tracing and webhook suites run.
This commit is contained in:
@@ -212,3 +212,303 @@ pub(crate) async fn metric_query(
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// ---- Traces (MON-10 to MON-17, MON-31, MON-32) ----
|
||||
|
||||
use common::telemetry::tracers::store::{MaybeTrace, decode_trace};
|
||||
use registry::schema::structs::{Search, Trace, TraceValue};
|
||||
use store::{
|
||||
IterateParams,
|
||||
search::{SearchFilter, SearchQuery, TracingSearchField},
|
||||
write::{BatchBuilder, SearchIndex, key::DeserializeBigEndian},
|
||||
};
|
||||
use trc::{AddContext, EventType, Key};
|
||||
|
||||
/// The trace's server-set fields (MON-14): the first event's time, the
|
||||
/// first `from`, every distinct `to`, and the message size or 0.
|
||||
fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
|
||||
let mut from = None;
|
||||
let mut to: Vec<String> = Vec::new();
|
||||
let mut size = None;
|
||||
let mut first_timestamp = None;
|
||||
for event in trace.events.iter() {
|
||||
first_timestamp.get_or_insert(event.timestamp.timestamp());
|
||||
for kv in event.key_values.iter() {
|
||||
let mut texts = Vec::new();
|
||||
match &kv.value {
|
||||
TraceValue::String(v) => texts.push(v.value.clone()),
|
||||
TraceValue::List(list) => {
|
||||
for item in list.value.iter() {
|
||||
if let TraceValue::String(v) = item {
|
||||
texts.push(v.value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
TraceValue::UnsignedInt(v) if kv.key == Key::Size => {
|
||||
size.get_or_insert(v.value);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
match kv.key {
|
||||
Key::From => {
|
||||
if from.is_none() {
|
||||
from = texts.into_iter().next();
|
||||
}
|
||||
}
|
||||
Key::To => {
|
||||
for text in texts {
|
||||
if !to.contains(&text) {
|
||||
to.push(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let timestamp = first_timestamp
|
||||
.unwrap_or_else(|| SnowflakeIdGenerator::to_timestamp(id) as i64);
|
||||
let mut value = trace.into_value();
|
||||
if let JmapValue::Object(obj) = &mut value {
|
||||
obj.insert_unchecked(
|
||||
Property::Timestamp,
|
||||
JmapValue::Str(UTCDateTime::from_timestamp(timestamp).to_string().into()),
|
||||
);
|
||||
obj.insert_unchecked(
|
||||
Property::From,
|
||||
match from {
|
||||
Some(from) => JmapValue::Str(from.into()),
|
||||
None => JmapValue::Null,
|
||||
},
|
||||
);
|
||||
obj.insert_unchecked(Property::To, JmapValue::Str(to.join(", ").into()));
|
||||
obj.insert_unchecked(Property::Size, JmapValue::Number(size.unwrap_or(0).into()));
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
/// The oldest trace id still visible (MON-17).
|
||||
async fn trace_floor(server: &common::Server) -> u64 {
|
||||
match common::telemetry::metrics::store::retention(server)
|
||||
.await
|
||||
.hold_traces_for
|
||||
{
|
||||
Some(keep) => SnowflakeIdGenerator::from_duration(keep.into_inner()).unwrap_or(0),
|
||||
None => 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_trace(server: &common::Server, id: u64) -> trc::Result<Option<Trace>> {
|
||||
if id < trace_floor(server).await {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(server
|
||||
.tracing_store()
|
||||
.get_value::<MaybeTrace>(ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(id))))
|
||||
.await?
|
||||
.and_then(|MaybeTrace(trace)| trace))
|
||||
}
|
||||
|
||||
/// `x:Trace/get`.
|
||||
pub(crate) async fn trace_get(
|
||||
mut get: RegistryGetResponse<'_>,
|
||||
) -> trc::Result<RegistryGetResponse<'_>> {
|
||||
assert_server_level(get.access_token)?;
|
||||
let server = get.server;
|
||||
if server.tracing_store().is_none() {
|
||||
if let Some(ids) = get.ids.take() {
|
||||
for id in ids {
|
||||
get.not_found(id);
|
||||
}
|
||||
}
|
||||
return Ok(get);
|
||||
}
|
||||
let ids = match get.ids.take() {
|
||||
Some(ids) => ids,
|
||||
None => trace_ids(server, 0, u64::MAX, false, None)
|
||||
.await?
|
||||
.into_iter()
|
||||
.take(server.core.jmap.get_max_objects)
|
||||
.map(Id::from)
|
||||
.collect(),
|
||||
};
|
||||
for id in ids {
|
||||
match read_trace(server, id.id()).await? {
|
||||
Some(trace) => get.insert(id, trace_to_value(id.id(), trace)),
|
||||
None => get.not_found(id),
|
||||
}
|
||||
}
|
||||
Ok(get)
|
||||
}
|
||||
|
||||
/// Trace ids in a range, newest first unless `ascending`, optionally only
|
||||
/// those whose opening event is `event`.
|
||||
async fn trace_ids(
|
||||
server: &common::Server,
|
||||
from_id: u64,
|
||||
to_id: u64,
|
||||
ascending: bool,
|
||||
event: Option<EventType>,
|
||||
) -> trc::Result<Vec<u64>> {
|
||||
let from_id = from_id.max(trace_floor(server).await);
|
||||
let mut ids = Vec::new();
|
||||
if from_id > to_id || server.tracing_store().is_none() {
|
||||
return Ok(ids);
|
||||
}
|
||||
let params = IterateParams::new(
|
||||
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(from_id))),
|
||||
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(to_id))),
|
||||
);
|
||||
let params = if ascending {
|
||||
params.ascending()
|
||||
} else {
|
||||
params.descending()
|
||||
};
|
||||
server
|
||||
.tracing_store()
|
||||
.iterate(params, |key, value| {
|
||||
let id = key.deserialize_be_u64(0)?;
|
||||
if let Some(trace) = decode_trace(value)
|
||||
&& event.is_none_or(|event| {
|
||||
trace.events.iter().next().is_some_and(|first| first.event == event)
|
||||
})
|
||||
{
|
||||
ids.push(id);
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// `x:Trace/query`: `event` (the opening event), `text` and `queueId`
|
||||
/// (through the search index, refused when trace search is off), and the
|
||||
/// timestamp comparisons.
|
||||
pub(crate) async fn trace_query(
|
||||
mut req: RegistryQueryResponse<'_>,
|
||||
) -> trc::Result<QueryResponseBuilder> {
|
||||
assert_server_level(req.access_token)?;
|
||||
let (mut from_id, mut to_id) = (0u64, u64::MAX);
|
||||
let mut event = None;
|
||||
let mut search: Vec<SearchFilter> = Vec::new();
|
||||
req.request.extract_filters(|property, op, value| match property {
|
||||
Property::Timestamp => {
|
||||
let Some(ts) = timestamp_of(&value) else {
|
||||
return false;
|
||||
};
|
||||
let at = SnowflakeIdGenerator::first_id_at;
|
||||
match op {
|
||||
RegistryFilterOp::GreaterThan => from_id = from_id.max(at(ts + 1)),
|
||||
RegistryFilterOp::GreaterEqualThan => from_id = from_id.max(at(ts)),
|
||||
RegistryFilterOp::LowerThan => to_id = to_id.min(at(ts).saturating_sub(1)),
|
||||
RegistryFilterOp::LowerEqualThan => {
|
||||
to_id = to_id.min(at(ts + 1).saturating_sub(1))
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
Property::Event => match value.as_str().and_then(EventType::parse) {
|
||||
Some(parsed) => {
|
||||
event = Some(parsed);
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
Property::Text => match value.as_str() {
|
||||
Some(text) => {
|
||||
search.push(SearchFilter::has_text(
|
||||
TracingSearchField::Keywords,
|
||||
text.to_lowercase(),
|
||||
nlp::language::Language::None,
|
||||
));
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
Property::QueueId => match value.as_str() {
|
||||
Some(queue_id) => {
|
||||
search.push(SearchFilter::eq(TracingSearchField::QueueId, queue_id.to_string()));
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
_ => false,
|
||||
})?;
|
||||
|
||||
let params = req
|
||||
.request
|
||||
.extract_parameters(req.server.core.jmap.query_max_results, None)?;
|
||||
if !matches!(params.sort_by, Property::Timestamp | Property::Id) {
|
||||
return Err(trc::JmapEvent::UnsupportedSort
|
||||
.into_err()
|
||||
.details("Traces sort by timestamp only"));
|
||||
}
|
||||
|
||||
let mut ids = trace_ids(req.server, from_id, to_id, params.sort_ascending, event).await?;
|
||||
if !search.is_empty() {
|
||||
let settings = req
|
||||
.server
|
||||
.registry()
|
||||
.object::<Search>(Id::singleton())
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
if !settings.index_telemetry {
|
||||
return Err(trc::JmapEvent::UnsupportedFilter
|
||||
.into_err()
|
||||
.details("Trace search is off (indexTelemetry)"));
|
||||
}
|
||||
let found = req
|
||||
.server
|
||||
.search_store()
|
||||
.query_global(SearchQuery::new(SearchIndex::Tracing).with_filters(search))
|
||||
.await?;
|
||||
let found = found.into_iter().collect::<std::collections::HashSet<_>>();
|
||||
ids.retain(|id| found.contains(id));
|
||||
}
|
||||
|
||||
let mut response = QueryResponseBuilder::new(
|
||||
ids.len(),
|
||||
req.server.core.jmap.query_max_results,
|
||||
State::Initial,
|
||||
&req.request,
|
||||
);
|
||||
for id in ids {
|
||||
if !response.add_id(Id::from(id)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// `x:Trace/set`: create and update are refused; destroy removes the trace
|
||||
/// and its search document (MON-32).
|
||||
pub(crate) async fn trace_set(
|
||||
mut set: crate::registry::mapping::RegistrySetResponse<'_>,
|
||||
) -> trc::Result<crate::registry::mapping::RegistrySetResponse<'_>> {
|
||||
assert_server_level(set.access_token)?;
|
||||
set.fail_all_create("Traces cannot be created");
|
||||
set.fail_all_update("Traces cannot be modified");
|
||||
let server = set.server;
|
||||
for id in std::mem::take(&mut set.destroy) {
|
||||
if read_trace(server, id.id()).await?.is_none() {
|
||||
set.response
|
||||
.not_destroyed
|
||||
.append(id, jmap_proto::error::set::SetError::not_found());
|
||||
continue;
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(ValueClass::Telemetry(TelemetryClass::Span(id.id())));
|
||||
server.tracing_store().write(batch.build_all()).await?;
|
||||
server
|
||||
.search_store()
|
||||
.unindex(
|
||||
SearchQuery::new(SearchIndex::Tracing)
|
||||
.with_filter(SearchFilter::eq(store::search::SearchField::Id, id.id())),
|
||||
)
|
||||
.await?;
|
||||
set.response.destroyed.push(id);
|
||||
}
|
||||
Ok(set)
|
||||
}
|
||||
|
||||
@@ -381,6 +381,9 @@ impl RegistryGet for Server {
|
||||
}
|
||||
ObjectType::Log => log_get(get).await.map(|get| get.into_response()),
|
||||
// inbuxa: monitoring history (MON-17, MON-31)
|
||||
ObjectType::Trace => crate::inbuxa::telemetry::trace_get(get)
|
||||
.await
|
||||
.map(|get| get.into_response()),
|
||||
ObjectType::Metric => crate::inbuxa::telemetry::metric_get(get)
|
||||
.await
|
||||
.map(|get| get.into_response()),
|
||||
|
||||
@@ -123,6 +123,14 @@ impl RegistryQuery for Server {
|
||||
.and_then(|response| response.build()),
|
||||
|
||||
// inbuxa: monitoring history (MON-17, MON-31)
|
||||
ObjectType::Trace => crate::inbuxa::telemetry::trace_query(RegistryQueryResponse {
|
||||
server: self,
|
||||
access_token,
|
||||
object_type,
|
||||
request,
|
||||
})
|
||||
.await
|
||||
.and_then(|response| response.build()),
|
||||
ObjectType::Metric => crate::inbuxa::telemetry::metric_query(RegistryQueryResponse {
|
||||
server: self,
|
||||
access_token,
|
||||
|
||||
@@ -881,7 +881,11 @@ impl RegistrySet for Server {
|
||||
.await
|
||||
.map(|set| set.into_response()),
|
||||
|
||||
ObjectType::Log | ObjectType::Metric | ObjectType::Trace | ObjectType::ClusterNode => {
|
||||
// inbuxa: MON-32: a trace can be destroyed, never created or changed
|
||||
ObjectType::Trace => crate::inbuxa::telemetry::trace_set(set)
|
||||
.await
|
||||
.map(|set| set.into_response()),
|
||||
ObjectType::Log | ObjectType::Metric | ObjectType::ClusterNode => {
|
||||
set.fail_all_create("Telemetry objects cannot be created");
|
||||
set.fail_all_update("Telemetry objects cannot be modified");
|
||||
set.fail_all_destroy("Telemetry objects cannot be deleted");
|
||||
|
||||
Reference in New Issue
Block a user