Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,376 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{smtp::session::TestSession, utils::server::TestServerBuilder};
|
||||
use ahash::AHashMap;
|
||||
use flate2::{Compression, Crc, write::GzEncoder};
|
||||
use mail_builder::{
|
||||
MessageBuilder,
|
||||
mime::{BodyPart, MimePart},
|
||||
};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::TaskStoreMaintenanceType,
|
||||
structs::{
|
||||
ArfExternalReport, DataRetention, DmarcExternalReport, Expression, MtaStageData,
|
||||
ReportSettings, Task, TaskStatus, TaskStoreMaintenance, TlsExternalReport,
|
||||
},
|
||||
},
|
||||
types::map::Map,
|
||||
};
|
||||
use std::{io::Write, time::Duration};
|
||||
|
||||
const MAX_REPORT_SIZE: i64 = 65536;
|
||||
|
||||
const DMARC_REPORT: &str = concat!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?><feedback><report_metadata>"#,
|
||||
r#"<org_name>Example</org_name><email>[email protected]</email>"#,
|
||||
r#"<report_id>1</report_id><date_range><begin>1</begin><end>2</end></date_range>"#,
|
||||
r#"</report_metadata><policy_published><domain>foobar.org</domain>"#,
|
||||
r#"</policy_published></feedback>"#
|
||||
);
|
||||
|
||||
fn report_message(content_type: &str, file_name: &str, payload: &[u8]) -> String {
|
||||
MessageBuilder::new()
|
||||
.from(("Reporter", "[email protected]"))
|
||||
.to("[email protected]")
|
||||
.subject("Report Domain: foobar.org")
|
||||
.body(MimePart::new(
|
||||
"multipart/report",
|
||||
BodyPart::Multipart(vec![
|
||||
MimePart::new("text/plain", BodyPart::Text("Report attached.".into())),
|
||||
MimePart::new(content_type, BodyPart::Binary(payload.into())).attachment(file_name),
|
||||
]),
|
||||
))
|
||||
.write_to_string()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn report_analyze() {
|
||||
let mut test = TestServerBuilder::new("smtp_analyze_report_test")
|
||||
.await
|
||||
.with_http_listener(19044)
|
||||
.await
|
||||
.capture_queue()
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let admin = test.account("admin");
|
||||
admin
|
||||
.registry_create_object(MtaStageData {
|
||||
max_messages: Expression {
|
||||
else_: "100".into(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(ReportSettings {
|
||||
inbound_report_addresses: Map::new(vec![
|
||||
"reports@*".to_string(),
|
||||
"*@dmarc.foobar.org".to_string(),
|
||||
"[email protected]".to_string(),
|
||||
]),
|
||||
inbound_report_forwarding: false,
|
||||
inbound_report_max_size: MAX_REPORT_SIZE,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(DataRetention {
|
||||
hold_mta_reports_for: Some(1u64.into()),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin.mta_no_auth().await;
|
||||
admin.mta_allow_non_fqdn().await;
|
||||
admin.mta_allow_relaying().await;
|
||||
admin.reload_settings().await;
|
||||
test.reload_core();
|
||||
test.expect_reload_settings().await;
|
||||
|
||||
// Create test message
|
||||
let mut session = test.new_mta_session();
|
||||
session.data.remote_ip_str = "10.0.0.1".into();
|
||||
session.eval_session_params().await;
|
||||
session.ehlo("mx.test.org").await;
|
||||
|
||||
let addresses = [
|
||||
"[email protected]",
|
||||
"[email protected]",
|
||||
"[email protected]",
|
||||
];
|
||||
let mut ac = 0;
|
||||
let mut total_reports_received: AHashMap<&str, usize> = AHashMap::new();
|
||||
for (test_name, num_tests) in [("arf", 5), ("dmarc", 5), ("tls", 2)] {
|
||||
for num_test in 1..=num_tests {
|
||||
*total_reports_received.entry(test_name).or_insert(0) += 1;
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&[addresses[ac % addresses.len()]],
|
||||
&format!("report:{test_name}{num_test}"),
|
||||
"250",
|
||||
)
|
||||
.await;
|
||||
test.assert_no_events();
|
||||
ac += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Report ingestion is asynchronous, poll until the reports are stored
|
||||
let admin = test.account("admin");
|
||||
for _ in 0..50 {
|
||||
if admin.registry_get_all::<DmarcExternalReport>().await.len()
|
||||
== total_reports_received["dmarc"]
|
||||
&& admin.registry_get_all::<TlsExternalReport>().await.len()
|
||||
== total_reports_received["tls"]
|
||||
&& admin.registry_get_all::<ArfExternalReport>().await.len()
|
||||
== total_reports_received["arf"]
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
// Purging the database shouldn't remove the reports
|
||||
admin
|
||||
.registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance {
|
||||
maintenance_type: TaskStoreMaintenanceType::PurgeData,
|
||||
shard_index: None,
|
||||
status: TaskStatus::now(),
|
||||
}))
|
||||
.await;
|
||||
test.wait_for_tasks().await;
|
||||
|
||||
// Make sure the reports are in the store
|
||||
assert_eq!(
|
||||
admin.registry_get_all::<DmarcExternalReport>().await.len(),
|
||||
total_reports_received["dmarc"]
|
||||
);
|
||||
assert_eq!(
|
||||
admin.registry_get_all::<TlsExternalReport>().await.len(),
|
||||
total_reports_received["tls"]
|
||||
);
|
||||
assert_eq!(
|
||||
admin.registry_get_all::<ArfExternalReport>().await.len(),
|
||||
total_reports_received["arf"]
|
||||
);
|
||||
|
||||
// Wait one second, purge, and make sure they are gone
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
admin
|
||||
.registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance {
|
||||
maintenance_type: TaskStoreMaintenanceType::PurgeData,
|
||||
shard_index: None,
|
||||
status: TaskStatus::now(),
|
||||
}))
|
||||
.await;
|
||||
test.wait_for_tasks().await;
|
||||
assert_eq!(
|
||||
admin.registry_get_all::<DmarcExternalReport>().await,
|
||||
vec![]
|
||||
);
|
||||
assert_eq!(admin.registry_get_all::<TlsExternalReport>().await, vec![]);
|
||||
assert_eq!(admin.registry_get_all::<ArfExternalReport>().await, vec![]);
|
||||
|
||||
// Reports that lie about their size or exceed the limit must not be ingested
|
||||
let attachment_name = "mx.test.org!foobar.org!1!2.xml";
|
||||
for payload in [
|
||||
report_message(
|
||||
"application/zip",
|
||||
&format!("{attachment_name}.zip"),
|
||||
&zip("report.xml", DMARC_REPORT.as_bytes(), None, Some(u32::MAX)),
|
||||
),
|
||||
report_message(
|
||||
"application/gzip",
|
||||
&format!("{attachment_name}.gz"),
|
||||
&gzip(&vec![b' '; MAX_REPORT_SIZE as usize * 2]),
|
||||
),
|
||||
] {
|
||||
session
|
||||
.send_message("[email protected]", &["[email protected]"], &payload, "250")
|
||||
.await;
|
||||
test.assert_no_events();
|
||||
}
|
||||
|
||||
// A report within the limit is still ingested
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&["[email protected]"],
|
||||
&report_message(
|
||||
"application/zip",
|
||||
&format!("{attachment_name}.zip"),
|
||||
&zip("report.xml", DMARC_REPORT.as_bytes(), None, None),
|
||||
),
|
||||
"250",
|
||||
)
|
||||
.await;
|
||||
test.assert_no_events();
|
||||
|
||||
let admin = test.account("admin");
|
||||
for _ in 0..50 {
|
||||
if !admin
|
||||
.registry_get_all::<DmarcExternalReport>()
|
||||
.await
|
||||
.is_empty()
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
admin.registry_get_all::<DmarcExternalReport>().await.len(),
|
||||
1
|
||||
);
|
||||
|
||||
// Redeliveries of a previously stored report must not be imported again
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&["[email protected]"],
|
||||
&report_message(
|
||||
"application/zip",
|
||||
&format!("{attachment_name}.zip"),
|
||||
&zip("report.xml", DMARC_REPORT.as_bytes(), None, None),
|
||||
),
|
||||
"250",
|
||||
)
|
||||
.await;
|
||||
test.assert_no_events();
|
||||
|
||||
let admin = test.account("admin");
|
||||
for _ in 0..10 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
assert_eq!(
|
||||
admin.registry_get_all::<DmarcExternalReport>().await.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
// Test delivery to non-report addresses
|
||||
session
|
||||
.send_message("[email protected]", &["[email protected]"], "test:no_dkim", "250")
|
||||
.await;
|
||||
test.expect_refresh().await;
|
||||
test.last_queued_message().await;
|
||||
|
||||
// Messages sent to a report address that contain no report must be delivered
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&["[email protected]"],
|
||||
concat!(
|
||||
"From: [email protected]\r\n",
|
||||
"To: [email protected]\r\n",
|
||||
"Subject: Your MX is refusing my connections\r\n",
|
||||
"\r\n",
|
||||
"Could you have a look at this?"
|
||||
),
|
||||
"250",
|
||||
)
|
||||
.await;
|
||||
let message = test.expect_message().await;
|
||||
assert_eq!(
|
||||
message.message.recipients.last().unwrap().address(),
|
||||
"[email protected]"
|
||||
);
|
||||
|
||||
// Reports addressed to both a report address and a regular mailbox are
|
||||
// discarded only for the report address
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&["[email protected]", "[email protected]"],
|
||||
&report_message(
|
||||
"application/zip",
|
||||
&format!("{attachment_name}.zip"),
|
||||
&zip("report.xml", DMARC_REPORT.as_bytes(), None, None),
|
||||
),
|
||||
"250",
|
||||
)
|
||||
.await;
|
||||
let message = test.expect_message().await;
|
||||
assert_eq!(
|
||||
message
|
||||
.message
|
||||
.recipients
|
||||
.iter()
|
||||
.map(|rcpt| rcpt.address())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["[email protected]"]
|
||||
);
|
||||
}
|
||||
|
||||
fn gzip(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(data).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
fn zip(
|
||||
name: &str,
|
||||
data: &[u8],
|
||||
compressed_size: Option<u32>,
|
||||
uncompressed_size: Option<u32>,
|
||||
) -> Vec<u8> {
|
||||
let mut crc = Crc::new();
|
||||
crc.update(data);
|
||||
let crc = crc.sum();
|
||||
let compressed_size = compressed_size.unwrap_or(data.len() as u32);
|
||||
let uncompressed_size = uncompressed_size.unwrap_or(data.len() as u32);
|
||||
let name = name.as_bytes();
|
||||
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&20u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&crc.to_le_bytes());
|
||||
out.extend_from_slice(&compressed_size.to_le_bytes());
|
||||
out.extend_from_slice(&uncompressed_size.to_le_bytes());
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(name);
|
||||
out.extend_from_slice(data);
|
||||
|
||||
let central_offset = out.len() as u32;
|
||||
out.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&20u16.to_le_bytes());
|
||||
out.extend_from_slice(&20u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&crc.to_le_bytes());
|
||||
out.extend_from_slice(&compressed_size.to_le_bytes());
|
||||
out.extend_from_slice(&uncompressed_size.to_le_bytes());
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u32.to_le_bytes());
|
||||
out.extend_from_slice(&0u32.to_le_bytes());
|
||||
out.extend_from_slice(name);
|
||||
|
||||
let central_size = out.len() as u32 - central_offset;
|
||||
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&1u16.to_le_bytes());
|
||||
out.extend_from_slice(&1u16.to_le_bytes());
|
||||
out.extend_from_slice(¢ral_size.to_le_bytes());
|
||||
out.extend_from_slice(¢ral_offset.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
|
||||
out
|
||||
}
|
||||
Reference in New Issue
Block a user