Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,493 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::AggregateTimestamp;
|
||||
use crate::{
|
||||
queue::RecipientDomain,
|
||||
reporting::{index::InternalReportIndex, send::MtaReportSend},
|
||||
};
|
||||
use common::{
|
||||
Server, USER_AGENT,
|
||||
config::smtp::{
|
||||
report::AggregateFrequency,
|
||||
resolver::{Mode, MxPattern, TlsaMatching},
|
||||
},
|
||||
ipc::{TlsEvent, ToHash},
|
||||
};
|
||||
use mail_auth::{
|
||||
flate2::{Compression, write::GzEncoder},
|
||||
mta_sts::{ReportUri, TlsRpt},
|
||||
report::tlsrpt::{FailureDetails, PolicyDetails},
|
||||
};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::TlsPolicyType,
|
||||
prelude::{ObjectType, Property},
|
||||
structs::{TlsFailureDetails, TlsInternalReport, TlsReport, TlsReportPolicy},
|
||||
},
|
||||
types::{EnumImpl, ObjectImpl, datetime::UTCDateTime},
|
||||
};
|
||||
use reqwest::header::CONTENT_TYPE;
|
||||
use std::fmt::Write;
|
||||
use std::{future::Future, sync::Arc, time::Duration};
|
||||
use store::{
|
||||
SerializeInfallible, ValueKey,
|
||||
registry::ObjectIdVersioned,
|
||||
write::{BatchBuilder, RegistryClass, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::{AddContext, OutgoingReportEvent};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TlsRptOptions {
|
||||
pub record: Arc<TlsRpt>,
|
||||
pub interval: AggregateFrequency,
|
||||
}
|
||||
|
||||
#[derive(Debug, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Serialize)]
|
||||
pub struct TlsFormat {
|
||||
pub rua: Vec<ReportUri>,
|
||||
pub policy: PolicyDetails,
|
||||
pub records: Vec<Option<FailureDetails>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
pub static TLS_HTTP_REPORT: parking_lot::Mutex<Vec<u8>> = parking_lot::Mutex::new(Vec::new());
|
||||
|
||||
pub trait TlsReporting: Sync + Send {
|
||||
fn send_tls_aggregate_report(
|
||||
&self,
|
||||
report_id: u64,
|
||||
) -> impl Future<Output = trc::Result<()>> + Send;
|
||||
|
||||
fn schedule_tls(&self, event: Box<TlsEvent>) -> impl Future<Output = ()> + Send;
|
||||
}
|
||||
|
||||
impl TlsReporting for Server {
|
||||
async fn send_tls_aggregate_report(&self, item_id: u64) -> trc::Result<()> {
|
||||
let object_id = ObjectType::TlsInternalReport.to_id();
|
||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
||||
|
||||
let Some(report) = self
|
||||
.store()
|
||||
.get_value::<TlsInternalReport>(ValueKey::from(key.clone()))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
// Delete report
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(key).clear(RegistryClass::PrimaryKey {
|
||||
object_id: object_id.into(),
|
||||
index_id: Property::Domain.to_id(),
|
||||
key: report.domain.as_bytes().to_vec(),
|
||||
});
|
||||
self.core
|
||||
.storage
|
||||
.data
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
let domain_name = report.domain.as_str();
|
||||
let event_from = report.report.date_range_start.timestamp() as u64;
|
||||
let event_to = report.report.date_range_end.timestamp() as u64;
|
||||
let span_id = self.inner.data.span_id_gen.generate();
|
||||
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::TlsAggregate),
|
||||
SpanId = span_id,
|
||||
ReportId = event_from,
|
||||
Domain = domain_name.to_string(),
|
||||
RangeFrom = trc::Value::Timestamp(event_from),
|
||||
RangeTo = trc::Value::Timestamp(event_to),
|
||||
);
|
||||
|
||||
// Generate report
|
||||
let exported_report = mail_auth::report::tlsrpt::TlsReport::from(report.report);
|
||||
let json = exported_report.to_json();
|
||||
let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default());
|
||||
let json = match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish())
|
||||
{
|
||||
Ok(report) => report,
|
||||
Err(err) => {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::SubmissionError),
|
||||
SpanId = span_id,
|
||||
Reason = err.to_string(),
|
||||
Details = "Failed to compress report"
|
||||
);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// Try delivering report over HTTP
|
||||
for uri in report.http_rua.as_slice() {
|
||||
{
|
||||
#[cfg(feature = "test_mode")]
|
||||
if uri == "https://127.0.0.1/tls" {
|
||||
TLS_HTTP_REPORT.lock().extend_from_slice(&json);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match self
|
||||
.core
|
||||
.smtp
|
||||
.tls_report_client
|
||||
.post(uri)
|
||||
.timeout(Duration::from_secs(2 * 60))
|
||||
.header(reqwest::header::USER_AGENT, USER_AGENT)
|
||||
.header(CONTENT_TYPE, "application/tlsrpt+gzip")
|
||||
.body(json.to_vec())
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => {
|
||||
if response.status().is_success() {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::HttpSubmission),
|
||||
SpanId = span_id,
|
||||
Url = uri.to_string(),
|
||||
Code = response.status().as_u16(),
|
||||
);
|
||||
|
||||
return Ok(());
|
||||
} else {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::SubmissionError),
|
||||
SpanId = span_id,
|
||||
Url = uri.to_string(),
|
||||
Code = response.status().as_u16(),
|
||||
Details = "Invalid HTTP response"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::SubmissionError),
|
||||
SpanId = span_id,
|
||||
Url = uri.to_string(),
|
||||
Reason = err.to_string(),
|
||||
Details = "HTTP submission error"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deliver report over SMTP
|
||||
if !report.mail_rua.is_empty() {
|
||||
let config = &self.core.smtp.report.tls;
|
||||
let from_addr = self
|
||||
.eval_if(&config.address, &RecipientDomain::new(domain_name), span_id)
|
||||
.await
|
||||
.unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string());
|
||||
let mut message = Vec::with_capacity(2048);
|
||||
let _ = exported_report.write_rfc5322_from_bytes(
|
||||
domain_name,
|
||||
&self
|
||||
.eval_if(
|
||||
&self.core.smtp.report.submitter,
|
||||
&RecipientDomain::new(domain_name),
|
||||
span_id,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|| "localhost".to_string()),
|
||||
(
|
||||
self.eval_if(&config.name, &RecipientDomain::new(domain_name), span_id)
|
||||
.await
|
||||
.unwrap_or_else(|| "Mail Delivery Subsystem".to_string())
|
||||
.as_str(),
|
||||
from_addr.as_str(),
|
||||
),
|
||||
report.mail_rua.iter().map(|v| v.as_str()),
|
||||
&json,
|
||||
&mut message,
|
||||
);
|
||||
|
||||
// Send report
|
||||
self.send_report(
|
||||
&from_addr,
|
||||
report.mail_rua.iter().map(|v| v.as_str()),
|
||||
message,
|
||||
&config.sign,
|
||||
false,
|
||||
span_id,
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::NoRecipientsFound),
|
||||
SpanId = span_id,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn schedule_tls(&self, event: Box<TlsEvent>) {
|
||||
let object_id = ObjectType::TlsInternalReport.to_id();
|
||||
let pk = ValueClass::Registry(RegistryClass::PrimaryKey {
|
||||
object_id: object_id.into(),
|
||||
index_id: Property::Domain.to_id(),
|
||||
key: event.domain.as_bytes().to_vec(),
|
||||
});
|
||||
let mut rety_count = 0;
|
||||
let policy_hash = event.policy.to_hash();
|
||||
|
||||
loop {
|
||||
// Find the report by domain name
|
||||
let mut batch = BatchBuilder::new();
|
||||
let report = match self
|
||||
.store()
|
||||
.get_value::<ObjectIdVersioned>(ValueKey::from(pk.clone()))
|
||||
.await
|
||||
{
|
||||
Ok(Some(object_id_v)) => {
|
||||
match self
|
||||
.store()
|
||||
.get_value::<TlsInternalReport>(ValueKey::from(ValueClass::Registry(
|
||||
RegistryClass::Item {
|
||||
object_id,
|
||||
item_id: object_id_v.object_id.id().id(),
|
||||
},
|
||||
)))
|
||||
.await
|
||||
{
|
||||
Ok(Some(report)) => Some((object_id_v, report)),
|
||||
Ok(None) => {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::NotFound),
|
||||
Id = object_id_v.object_id.id().id(),
|
||||
CausedBy = trc::location!(),
|
||||
Details = "Failed to find TLS report for domain"
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.caused_by(trc::location!())
|
||||
.details("Failed to query registry for TLS report")
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => None,
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.caused_by(trc::location!())
|
||||
.details("Failed to query registry for TLS report")
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Create report if missing
|
||||
let config = &self.core.smtp.report.tls;
|
||||
let (item_id, mut report) = if let Some((mut object_id_v, report)) = report {
|
||||
batch.assert_value(pk.clone(), AssertValue::U32(object_id_v.version));
|
||||
object_id_v.version += 1;
|
||||
batch.set(pk.clone(), object_id_v.serialize());
|
||||
|
||||
(object_id_v.object_id.id().id(), report)
|
||||
} else {
|
||||
let item_id = self.inner.data.queue_id_gen.generate();
|
||||
let date_range_start = UTCDateTime::now();
|
||||
let date_range_end = UTCDateTime::from_timestamp(
|
||||
date_range_start.timestamp() + event.interval.as_secs() as i64,
|
||||
);
|
||||
|
||||
let report = TlsInternalReport {
|
||||
created_at: date_range_start,
|
||||
deliver_at: date_range_end,
|
||||
domain: event.domain.clone(),
|
||||
report: TlsReport {
|
||||
report_id: format!("{}_{policy_hash}", date_range_start.timestamp()),
|
||||
organization_name: self
|
||||
.eval_if::<String, _>(
|
||||
&config.org_name,
|
||||
&RecipientDomain::new(&event.domain),
|
||||
event.span_id,
|
||||
)
|
||||
.await
|
||||
.clone(),
|
||||
contact_info: self
|
||||
.eval_if::<String, _>(
|
||||
&config.contact_info,
|
||||
&RecipientDomain::new(&event.domain),
|
||||
event.span_id,
|
||||
)
|
||||
.await
|
||||
.clone(),
|
||||
date_range_end,
|
||||
date_range_start,
|
||||
policies: Default::default(),
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
report.write_ops(&mut batch, item_id, true);
|
||||
|
||||
(item_id, report)
|
||||
};
|
||||
|
||||
let policy = if let Some(policy) = report
|
||||
.policy_identifiers
|
||||
.as_slice()
|
||||
.iter()
|
||||
.position(|id| *id == policy_hash)
|
||||
.and_then(|idx| report.report.policies.0.inner.get_mut(idx))
|
||||
{
|
||||
&mut policy.value
|
||||
} else {
|
||||
// Create policy
|
||||
let mut policy = TlsReportPolicy {
|
||||
policy_type: TlsPolicyType::NoPolicyFound,
|
||||
policy_domain: report.domain.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match &event.policy {
|
||||
common::ipc::PolicyType::Tlsa(tlsa) => {
|
||||
policy.policy_type = TlsPolicyType::Tlsa;
|
||||
if let Some(tlsa) = tlsa {
|
||||
for entry in &tlsa.entries {
|
||||
policy.policy_strings.push(format!(
|
||||
"{} {} {} {}",
|
||||
if entry.is_end_entity { 3 } else { 2 },
|
||||
i32::from(entry.is_spki),
|
||||
match entry.matching {
|
||||
TlsaMatching::Full => 0,
|
||||
TlsaMatching::Sha256 => 1,
|
||||
TlsaMatching::Sha512 => 2,
|
||||
},
|
||||
entry.data.iter().fold(
|
||||
String::with_capacity(64),
|
||||
|mut s, b| {
|
||||
write!(s, "{b:02X}").ok();
|
||||
s
|
||||
}
|
||||
)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
common::ipc::PolicyType::Sts(sts) => {
|
||||
policy.policy_type = TlsPolicyType::Sts;
|
||||
if let Some(sts) = sts {
|
||||
policy.policy_strings.push("version: STSv1".to_string());
|
||||
policy.policy_strings.push(format!(
|
||||
"mode: {}",
|
||||
match sts.mode {
|
||||
Mode::Enforce => "enforce",
|
||||
Mode::Testing => "testing",
|
||||
Mode::None => "none",
|
||||
}
|
||||
));
|
||||
policy
|
||||
.policy_strings
|
||||
.push(format!("max_age: {}", sts.max_age));
|
||||
for mx in &sts.mx {
|
||||
let mx = match mx {
|
||||
MxPattern::Equals(mx) => mx.to_string(),
|
||||
MxPattern::StartsWith(mx) => format!("*.{mx}"),
|
||||
};
|
||||
policy.policy_strings.push(format!("mx: {mx}"));
|
||||
policy.mx_hosts.push(mx);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
for rua in &event.tls_record.rua {
|
||||
match rua {
|
||||
ReportUri::Mail(mail) => {
|
||||
report.mail_rua.push(mail.clone());
|
||||
}
|
||||
ReportUri::Http(uri) => {
|
||||
report.http_rua.push(uri.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.policy_identifiers.push(policy_hash);
|
||||
report.report.policies.push(policy);
|
||||
&mut report.report.policies.0.inner.last_mut().unwrap().value
|
||||
};
|
||||
|
||||
// Add failure details
|
||||
if let Some(mut failure) = event.failure.clone().map(TlsFailureDetails::from) {
|
||||
if let Some(idx) = policy
|
||||
.failure_details
|
||||
.0
|
||||
.inner
|
||||
.iter()
|
||||
.position(|d| d.value.eq_except_count(&failure))
|
||||
{
|
||||
policy.failure_details.0.inner[idx]
|
||||
.value
|
||||
.failed_session_count += 1;
|
||||
} else {
|
||||
failure.failed_session_count = 1;
|
||||
policy.failure_details.push(failure);
|
||||
}
|
||||
|
||||
policy.total_failed_sessions += 1;
|
||||
} else {
|
||||
policy.total_successful_sessions += 1;
|
||||
}
|
||||
|
||||
// Write entry
|
||||
let report_bytes = report.to_pickled_vec();
|
||||
let max_report_size = self
|
||||
.eval_if(
|
||||
&config.max_size,
|
||||
&RecipientDomain::new(&event.domain),
|
||||
event.span_id,
|
||||
)
|
||||
.await
|
||||
.unwrap_or(5 * 1024 * 1024);
|
||||
if max_report_size != 0 && report_bytes.len() > max_report_size {
|
||||
trc::event!(
|
||||
OutgoingReport(OutgoingReportEvent::MaxSizeExceeded),
|
||||
SpanId = event.span_id,
|
||||
Domain = event.domain.clone(),
|
||||
Details = report_bytes.len(),
|
||||
Limit = max_report_size,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
batch.set(
|
||||
ValueClass::Registry(RegistryClass::Item { object_id, item_id }),
|
||||
report_bytes,
|
||||
);
|
||||
|
||||
match self.core.storage.data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
break;
|
||||
}
|
||||
Err(err) => {
|
||||
if err.is_assertion_failure() && rety_count < 3 {
|
||||
rety_count += 1;
|
||||
continue;
|
||||
}
|
||||
trc::error!(
|
||||
err.caused_by(trc::location!())
|
||||
.details("Failed to write TLS report")
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user