inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 3 Wiki Activity

Import upstream v0.16.22, stripped

Browse Source 
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
jcoffey-dev
2026-09-18 10:21:56 -07:00
commit 7dae9b29fd
1650 changed files with 485521 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/smtp/src/queue/dsn.rs
+669
View File
@@ -0,0 +1,669 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::spool::SmtpSpool;
use super::{
Error, ErrorDetails, HostResponse, Message, MessageSource, QueueEnvelope, RCPT_DSN_SENT,
Recipient, Status,
};
use crate::inbound::dkim::DkimSign;
use crate::queue::spool::QueueParams;
use crate::queue::{MessageWrapper, UnexpectedResponse};
use common::Server;
use mail_builder::MessageBuilder;
use mail_builder::headers::HeaderType;
use mail_builder::headers::content_type::ContentType;
use mail_builder::mime::{BodyPart, MimePart, make_boundary};
use mail_parser::DateTime;
use smtp_proto::{
RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Response,
};
use std::fmt::Write;
use std::future::Future;
use store::write::now;
pub trait SendDsn: Sync + Send {
fn send_dsn(&self, message: &mut MessageWrapper) -> impl Future<Output = ()> + Send;
fn log_dsn(&self, message: &MessageWrapper) -> impl Future<Output = ()> + Send;
}
impl SendDsn for Server {
async fn send_dsn(&self, message: &mut MessageWrapper) {
// Send DSN events
self.log_dsn(message).await;
if !message.message.return_path.is_empty() {
// Build DSN
if let Some(dsn) = message.build_dsn(self).await {
let mut dsn_message = self.new_message("", MessageSource::Dsn, message.span_id);
dsn_message
.expand_and_add_recipient(message.message.return_path.as_ref(), self)
.await;
// Queue DSN
let dkim_signers = self
.eval_signers(
&self.core.smtp.queue.dsn.sign,
&message.message,
message.span_id,
)
.await;
dsn_message
.queue(
QueueParams::new(&dsn, message.span_id, self)
.with_dkim_signers(dkim_signers),
)
.await;
}
} else {
// Handle double bounce
message.handle_double_bounce();
}
// Update next DSN notify times
message.update_next_dsn(self).await;
}
async fn log_dsn(&self, message: &MessageWrapper) {
let now = now();
for rcpt in &message.message.recipients {
if rcpt.has_flag(RCPT_DSN_SENT) {
continue;
}
match &rcpt.status {
Status::Completed(response) => {
trc::event!(
Delivery(trc::DeliveryEvent::DsnSuccess),
SpanId = message.span_id,
To = rcpt.address.clone(),
Hostname = response.hostname.clone(),
Code = response.response.code,
Details = response.response.message.to_string(),
);
}
Status::TemporaryFailure(response) if rcpt.notify.due <= now => {
trc::event!(
Delivery(trc::DeliveryEvent::DsnTempFail),
SpanId = message.span_id,
To = rcpt.address.clone(),
Hostname = response.entity.clone(),
Details = response.details.to_string(),
NextRetry = trc::Value::Timestamp(rcpt.retry.due),
Expires = rcpt
.expiration_time(message.message.created)
.map(trc::Value::Timestamp),
Total = rcpt.retry.inner,
);
}
Status::PermanentFailure(response) => {
trc::event!(
Delivery(trc::DeliveryEvent::DsnPermFail),
SpanId = message.span_id,
To = rcpt.address.clone(),
Hostname = response.entity.clone(),
Details = response.details.to_string(),
Total = rcpt.retry.inner,
);
}
Status::Scheduled if rcpt.notify.due <= now => {
trc::event!(
Delivery(trc::DeliveryEvent::DsnTempFail),
SpanId = message.span_id,
To = rcpt.address.clone(),
Details = "Concurrency limited",
NextRetry = trc::Value::Timestamp(rcpt.retry.due),
Expires = rcpt
.expiration_time(message.message.created)
.map(trc::Value::Timestamp),
Total = rcpt.retry.inner,
);
}
_ => continue,
}
}
}
}
const MAX_HEADER_SIZE: usize = 4096;
impl MessageWrapper {
pub async fn build_dsn(&mut self, server: &Server) -> Option<Vec<u8>> {
let config = &server.core.smtp.queue;
let now = now();
let mut txt_success = String::new();
let mut txt_delay = String::new();
let mut txt_failed = String::new();
let mut dsn = String::new();
for rcpt in &mut self.message.recipients {
if rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) {
continue;
}
match &rcpt.status {
Status::Completed(response) => {
rcpt.flags |= RCPT_DSN_SENT;
if !rcpt.has_flag(RCPT_NOTIFY_SUCCESS) {
continue;
}
rcpt.write_dsn(&mut dsn);
rcpt.status.write_dsn(&mut dsn);
response.write_dsn_text(&rcpt.address, &mut txt_success);
}
Status::TemporaryFailure(response)
if rcpt.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) =>
{
rcpt.write_dsn(&mut dsn);
rcpt.status.write_dsn(&mut dsn);
rcpt.write_dsn_will_retry_until(self.message.created, &mut dsn);
response.write_dsn_text(&rcpt.address, &mut txt_delay);
}
Status::PermanentFailure(response) => {
rcpt.flags |= RCPT_DSN_SENT;
if !rcpt.has_flag(RCPT_NOTIFY_FAILURE) {
continue;
}
rcpt.write_dsn(&mut dsn);
rcpt.status.write_dsn(&mut dsn);
response.write_dsn_text(&rcpt.address, &mut txt_failed);
}
Status::Scheduled if rcpt.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) => {
// This case should not happen under normal circumstances
rcpt.write_dsn(&mut dsn);
rcpt.status.write_dsn(&mut dsn);
rcpt.write_dsn_will_retry_until(self.message.created, &mut dsn);
ErrorDetails {
entity: "localhost".into(),
details: Error::ConcurrencyLimited,
}
.write_dsn_text(&rcpt.address, &mut txt_delay);
}
_ => continue,
}
dsn.push_str("\r\n");
}
let txt_len = txt_success.len() + txt_delay.len() + txt_failed.len();
if txt_len == 0 {
return None;
}
let has_success = !txt_success.is_empty();
let has_delay = !txt_delay.is_empty();
let has_failure = !txt_failed.is_empty();
let mut txt = String::with_capacity(txt_len + 128);
let (subject, is_mixed) = if has_success && !has_delay && !has_failure {
txt.push_str(
"Your message has been successfully delivered to the following recipients:\r\n\r\n",
);
("Successfully delivered message", false)
} else if has_delay && !has_success && !has_failure {
txt.push_str("There was a temporary problem delivering your message to the following recipients:\r\n\r\n");
("Warning: Delay in message delivery", false)
} else if has_failure && !has_success && !has_delay {
txt.push_str(
"Your message could not be delivered to the following recipients:\r\n\r\n",
);
("Failed to deliver message", false)
} else if has_success {
txt.push_str("Your message has been partially delivered:\r\n\r\n");
("Partially delivered message", true)
} else {
txt.push_str("Your message could not be delivered to some recipients:\r\n\r\n");
(
"Warning: Temporary and permanent failures during message delivery",
true,
)
};
if has_success {
if is_mixed {
txt.push_str(
" ----- Delivery to the following addresses was successful -----\r\n",
);
}
txt.push_str(&txt_success);
txt.push_str("\r\n");
}
if has_delay {
if is_mixed {
txt.push_str(
" ----- There was a temporary problem delivering to these addresses -----\r\n",
);
}
txt.push_str(&txt_delay);
txt.push_str("\r\n");
}
if has_failure {
if is_mixed {
txt.push_str(" ----- Delivery to the following addresses failed -----\r\n");
}
txt.push_str(&txt_failed);
txt.push_str("\r\n");
}
// Obtain hostname and sender addresses
let from_name = server
.eval_if(&config.dsn.name, &self.message, self.span_id)
.await
.unwrap_or_else(|| String::from("Mail Delivery Subsystem"));
let from_addr = server
.eval_if(&config.dsn.address, &self.message, self.span_id)
.await
.unwrap_or_else(|| String::from("MAILER-DAEMON@localhost"));
let reporting_mta = server
.eval_if(
&server.core.smtp.report.submitter,
&self.message,
self.span_id,
)
.await
.unwrap_or_else(|| String::from("localhost"));
// Prepare DSN
let mut dsn_header = String::with_capacity(dsn.len() + 128);
self.message
.write_dsn_headers(&mut dsn_header, &reporting_mta);
let dsn = dsn_header + dsn.as_str();
// Fetch up to MAX_HEADER_SIZE bytes of message headers
let headers = match server
.blob_store()
.get_blob(self.message.blob_hash.as_slice(), 0..MAX_HEADER_SIZE)
.await
{
Ok(Some(mut buf)) => {
let mut prev_ch = 0;
let mut last_lf = buf.len();
for (pos, &ch) in buf.iter().enumerate() {
match ch {
b'\n' => {
last_lf = pos + 1;
if prev_ch != b'\n' {
prev_ch = ch;
} else {
break;
}
}
b'\r' => (),
0 => break,
_ => {
prev_ch = ch;
}
}
}
if last_lf < MAX_HEADER_SIZE {
buf.truncate(last_lf);
}
String::from_utf8(buf).unwrap_or_default()
}
Ok(None) => {
trc::event!(
Queue(trc::QueueEvent::BlobNotFound),
SpanId = self.span_id,
BlobId = self.message.blob_hash.to_hex(),
CausedBy = trc::location!()
);
String::new()
}
Err(err) => {
trc::error!(
err.span_id(self.span_id)
.details("Failed to fetch blobId")
.caused_by(trc::location!())
);
String::new()
}
};
// Build message
MessageBuilder::new()
.from((from_name.as_str(), from_addr.as_str()))
.header(
"To",
HeaderType::Text(self.message.return_path.as_ref().into()),
)
.header("Auto-Submitted", HeaderType::Text("auto-generated".into()))
.message_id(format!("{}@{}", make_boundary("."), reporting_mta))
.subject(subject)
.body(MimePart::new(
ContentType::new("multipart/report").attribute("report-type", "delivery-status"),
BodyPart::Multipart(vec![
MimePart::new(ContentType::new("text/plain"), BodyPart::Text(txt.into())),
MimePart::new(
ContentType::new("message/delivery-status"),
BodyPart::Text(dsn.into()),
),
MimePart::new(
ContentType::new("message/rfc822"),
BodyPart::Text(headers.into()),
),
]),
))
.write_to_vec()
.unwrap_or_default()
.into()
}
pub async fn update_next_dsn(&mut self, server: &Server) {
let now = now();
let mut notify_changes = Vec::new();
for (rcpt_idx, rcpt) in self.message.recipients.iter().enumerate() {
if matches!(
&rcpt.status,
Status::TemporaryFailure(_) | Status::Scheduled
) && rcpt.notify.due <= now
{
let envelope = QueueEnvelope::new(&self.message, rcpt);
let queue_id = server
.eval_if::<String, _>(&server.core.smtp.queue.queue, &envelope, self.span_id)
.await
.unwrap_or_else(|| "default".to_string());
let queue = server.get_queue_or_default(&queue_id, self.span_id);
if let Some(next_notify) =
queue.notify.get((rcpt.notify.inner + 1) as usize).copied()
{
notify_changes.push((rcpt_idx, 1, now + next_notify));
} else {
notify_changes.push((rcpt_idx, 0, u64::MAX));
}
}
}
for (rcpt_idx, inner, due) in notify_changes {
let rcpt = &mut self.message.recipients[rcpt_idx];
rcpt.notify.inner += inner;
rcpt.notify.due = due;
}
}
fn handle_double_bounce(&mut self) {
let mut is_double_bounce = Vec::with_capacity(0);
let now = now();
for rcpt in &mut self.message.recipients {
if !rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER)
&& let Status::PermanentFailure(err) = &rcpt.status
{
rcpt.flags |= RCPT_DSN_SENT;
let mut dsn = String::new();
err.write_dsn_text(&rcpt.address, &mut dsn);
is_double_bounce.push(dsn);
}
if rcpt.notify.due <= now {
rcpt.notify.due = rcpt
.expiration_time(self.message.created)
.map(|d| d + 10)
.unwrap_or(u64::MAX);
}
}
if !is_double_bounce.is_empty() {
trc::event!(
Delivery(trc::DeliveryEvent::DoubleBounce),
SpanId = self.span_id,
To = is_double_bounce
);
}
}
}
impl HostResponse<Box<str>> {
fn write_dsn_text(&self, addr: &str, dsn: &mut String) {
let _ = write!(
dsn,
"<{}> (delivered to '{}' with code {} ({}.{}.{}) '",
addr,
self.hostname,
self.response.code,
self.response.esc[0],
self.response.esc[1],
self.response.esc[2]
);
self.response.write_response(dsn);
dsn.push_str("')\r\n");
}
}
impl UnexpectedResponse {
fn write_dsn_text(&self, host: &str, addr: &str, dsn: &mut String) {
let _ = write!(dsn, "<{addr}> (host '{host}' rejected ");
if !self.command.is_empty() {
let _ = write!(dsn, "command '{}'", self.command);
} else {
dsn.push_str("transaction");
}
let _ = write!(
dsn,
" with code {} ({}.{}.{}) '",
self.response.code, self.response.esc[0], self.response.esc[1], self.response.esc[2]
);
self.response.write_response(dsn);
dsn.push_str("')\r\n");
}
}
impl ErrorDetails {
fn write_dsn_text(&self, addr: &str, dsn: &mut String) {
let entity = self.entity.as_ref();
match &self.details {
Error::UnexpectedResponse(response) => {
response.write_dsn_text(entity, addr, dsn);
}
Error::DnsError(err) => {
let _ = write!(dsn, "<{addr}> (failed to lookup '{entity}': {err})\r\n",);
}
Error::ConnectionError(details) => {
let _ = write!(
dsn,
"<{addr}> (connection to '{entity}' failed: {details})\r\n",
);
}
Error::TlsError(details) => {
let _ = write!(dsn, "<{addr}> (TLS error from '{entity}': {details})\r\n",);
}
Error::DaneError(details) => {
let _ = write!(
dsn,
"<{addr}> (DANE failed to authenticate '{entity}': {details})\r\n",
);
}
Error::MtaStsError(details) => {
let _ = write!(
dsn,
"<{addr}> (MTA-STS failed to authenticate '{entity}': {details})\r\n",
);
}
Error::RateLimited => {
let _ = write!(dsn, "<{addr}> (rate limited)\r\n");
}
Error::ConcurrencyLimited => {
let _ = write!(
dsn,
"<{addr}> (too many concurrent connections to remote server)\r\n",
);
}
Error::Io(err) => {
let _ = write!(dsn, "<{addr}> (queue error: {err})\r\n");
}
}
}
}
impl Message {
fn write_dsn_headers(&self, dsn: &mut String, reporting_mta: &str) {
let _ = write!(dsn, "Reporting-MTA: dns;{reporting_mta}\r\n");
dsn.push_str("Arrival-Date: ");
dsn.push_str(&DateTime::from_timestamp(self.created as i64).to_rfc822());
dsn.push_str("\r\n");
if let Some(env_id) = &self.env_id {
let _ = write!(dsn, "Original-Envelope-Id: {env_id}\r\n");
}
dsn.push_str("\r\n");
}
}
impl Recipient {
fn write_dsn(&self, dsn: &mut String) {
if let Some(orcpt) = &self.orcpt {
let _ = write!(dsn, "Original-Recipient: rfc822;{orcpt}\r\n");
}
let _ = write!(dsn, "Final-Recipient: rfc822;{}\r\n", self.address);
}
fn write_dsn_will_retry_until(&self, created: u64, dsn: &mut String) {
if let Some(expires) = self.expiration_time(created)
&& expires > now()
{
dsn.push_str("Will-Retry-Until: ");
dsn.push_str(&DateTime::from_timestamp(expires as i64).to_rfc822());
dsn.push_str("\r\n");
}
}
}
impl<T, E> Status<T, E> {
pub fn into_permanent(self) -> Self {
match self {
Status::TemporaryFailure(v) => Status::PermanentFailure(v),
v => v,
}
}
pub fn into_temporary(self) -> Self {
match self {
Status::PermanentFailure(err) => Status::TemporaryFailure(err),
other => other,
}
}
pub fn is_permanent(&self) -> bool {
matches!(self, Status::PermanentFailure(_))
}
fn write_dsn_action(&self, dsn: &mut String) {
dsn.push_str("Action: ");
dsn.push_str(match self {
Status::Completed(_) => "delivered",
Status::PermanentFailure(_) => "failed",
Status::TemporaryFailure(_) | Status::Scheduled => "delayed",
});
dsn.push_str("\r\n");
}
}
impl Status<HostResponse<Box<str>>, ErrorDetails> {
fn write_dsn(&self, dsn: &mut String) {
self.write_dsn_action(dsn);
self.write_dsn_status(dsn);
self.write_dsn_diagnostic(dsn);
self.write_dsn_remote_mta(dsn);
}
fn write_dsn_status(&self, dsn: &mut String) {
dsn.push_str("Status: ");
match self {
Status::Completed(response) => {
response.response.write_dsn_status(dsn);
}
Status::TemporaryFailure(err) | Status::PermanentFailure(err) => {
if let Error::UnexpectedResponse(response) = &err.details {
response.response.write_dsn_status(dsn);
} else {
dsn.push_str(if matches!(self, Status::PermanentFailure(_)) {
"5.0.0"
} else {
"4.0.0"
});
}
}
Status::Scheduled => {
dsn.push_str("4.0.0");
}
}
dsn.push_str("\r\n");
}
fn write_dsn_remote_mta(&self, dsn: &mut String) {
match self {
Status::Completed(response) => {
dsn.push_str("Remote-MTA: dns;");
dsn.push_str(&response.hostname);
dsn.push_str("\r\n");
}
Status::TemporaryFailure(err) | Status::PermanentFailure(err) => match &err.details {
Error::UnexpectedResponse(_)
| Error::ConnectionError(_)
| Error::TlsError(_)
| Error::DaneError(_) => {
dsn.push_str("Remote-MTA: dns;");
dsn.push_str(&err.entity);
dsn.push_str("\r\n");
}
_ => (),
},
Status::Scheduled => (),
}
}
fn write_dsn_diagnostic(&self, dsn: &mut String) {
if let Status::PermanentFailure(err) | Status::TemporaryFailure(err) = self
&& let Error::UnexpectedResponse(response) = &err.details
{
response.response.write_dsn_diagnostic(dsn);
}
}
}
impl WriteDsn for Response<Box<str>> {
fn write_dsn_status(&self, dsn: &mut String) {
if self.esc[0] > 0 {
let _ = write!(dsn, "{}.{}.{}", self.esc[0], self.esc[1], self.esc[2]);
} else {
let _ = write!(
dsn,
"{}.{}.{}",
self.code / 100,
(self.code / 10) % 10,
self.code % 10
);
}
}
fn write_dsn_diagnostic(&self, dsn: &mut String) {
let _ = write!(dsn, "Diagnostic-Code: smtp;{} ", self.code);
self.write_response(dsn);
dsn.push_str("\r\n");
}
fn write_response(&self, dsn: &mut String) {
for ch in self.message.chars() {
if ch != '\n' && ch != '\r' {
dsn.push(ch);
}
}
}
}
trait WriteDsn {
fn write_dsn_status(&self, dsn: &mut String);
fn write_dsn_diagnostic(&self, dsn: &mut String);
fn write_response(&self, dsn: &mut String);
}
crates/smtp/src/queue/manager.rs
+501
View File
@@ -0,0 +1,501 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::{Message, QueueId, Status, spool::SmtpSpool};
use crate::queue::{
Recipient,
spool::{INFINITE_LOCK, LOCK_EXPIRY, QUEUE_REFRESH},
};
use ahash::AHashMap;
use common::{
BuildServer, Inner,
config::smtp::queue::{QueueExpiry, QueueName},
ipc::{QueueEvent, QueueEventStatus},
};
use rand::{RngExt, seq::SliceRandom};
use std::{
collections::hash_map::Entry,
sync::{Arc, atomic::Ordering},
time::{Duration, Instant},
};
use store::write::now;
use tokio::sync::mpsc;
pub struct Queue {
pub core: Arc<Inner>,
pub locked: AHashMap<(QueueId, QueueName), LockedMessage>,
pub locked_revision: u64,
pub stats: AHashMap<QueueName, QueueStats>,
pub next_refresh: Instant,
pub rx: mpsc::Receiver<QueueEvent>,
pub is_paused: bool,
pub scan_from: u64,
pub scan_ceiling: u64,
pub has_pending_work: bool,
pub pending_refresh: bool,
pub urgent_refresh: bool,
pub last_scan: Instant,
pub last_full_scan: Instant,
}
#[derive(Debug)]
pub struct QueueStats {
pub in_flight: usize,
pub max_in_flight: usize,
pub budget: usize,
pub last_warning: Instant,
}
#[derive(Debug)]
pub struct LockedMessage {
pub expires: u64,
pub revision: u64,
pub due: u64,
}
impl SpawnQueue for mpsc::Receiver<QueueEvent> {
fn spawn(self, core: Arc<Inner>) {
tokio::spawn(async move {
Queue::new(core, self).start().await;
});
}
}
const BACK_PRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(60);
const MIN_SCAN_INTERVAL: Duration = Duration::from_millis(100);
const FULL_SCAN_INTERVAL: Duration = Duration::from_secs(QUEUE_REFRESH / 2);
impl Queue {
pub fn new(core: Arc<Inner>, rx: mpsc::Receiver<QueueEvent>) -> Self {
let now = Instant::now();
Queue {
core,
locked: AHashMap::with_capacity(128),
locked_revision: 0,
stats: AHashMap::new(),
next_refresh: now + Duration::from_secs(1),
is_paused: false,
rx,
scan_from: 0,
scan_ceiling: u64::MAX,
has_pending_work: false,
pending_refresh: false,
urgent_refresh: false,
last_scan: now.checked_sub(MIN_SCAN_INTERVAL).unwrap_or(now),
last_full_scan: now,
}
}
pub async fn start(&mut self) {
trc::event!(Queue(trc::QueueEvent::Started));
loop {
let mut refresh_queue;
match tokio::time::timeout(
self.next_refresh.duration_since(Instant::now()),
self.rx.recv(),
)
.await
{
Ok(Some(event)) => {
refresh_queue = self.handle_event(event).await;
while let Ok(event) = self.rx.try_recv() {
refresh_queue = self.handle_event(event).await || refresh_queue;
}
}
Err(_) => {
refresh_queue = true;
self.urgent_refresh = true;
}
Ok(None) => {
break;
}
};
if self.is_paused {
self.next_refresh = Instant::now() + Duration::from_secs(86400);
continue;
}
self.pending_refresh |= refresh_queue;
if !self.pending_refresh && self.next_refresh > Instant::now() {
continue;
}
// Coalesce bursts of worker notifications into a single scan
let scan_at = self.last_scan + MIN_SCAN_INTERVAL;
if !self.urgent_refresh && scan_at > Instant::now() {
self.next_refresh = scan_at;
continue;
}
if self.scan_from != 0 && self.last_full_scan.elapsed() >= FULL_SCAN_INTERVAL {
self.scan_from = 0;
}
if self.scan_from == 0 {
self.last_full_scan = Instant::now();
}
let scan_floor = self.scan_from;
self.pending_refresh = false;
self.urgent_refresh = false;
// Process queue events
let server = self.core.build_server();
let mut queue_events = server.next_event(self).await;
self.last_scan = Instant::now();
if queue_events.messages.len() > 3 {
queue_events.messages.shuffle(&mut rand::rng());
}
// A truncated scan left events behind
let now = now();
self.has_pending_work = self.scan_ceiling != u64::MAX;
for queue_event in &queue_events.messages {
// A message may hold more than one event key, dispatch it only once
if self
.locked
.get(&(queue_event.queue_id, queue_event.queue_name))
.is_some_and(|locked| locked.expires > now)
{
continue;
}
// Fetch queue stats
let stats = match self.stats.get_mut(&queue_event.queue_name) {
Some(stats) => stats,
None => {
let queue_config =
server.get_virtual_queue_or_default(&queue_event.queue_name);
self.stats.insert(
queue_event.queue_name,
QueueStats::new(queue_config.threads),
);
self.stats.get_mut(&queue_event.queue_name).unwrap()
}
};
// Enforce concurrency limits
if stats.has_capacity() {
// Deliver message
stats.in_flight += 1;
self.locked.insert(
(queue_event.queue_id, queue_event.queue_name),
LockedMessage {
expires: now + INFINITE_LOCK,
revision: self.locked_revision,
due: queue_event.due,
},
);
queue_event.try_deliver(server.clone());
} else {
if stats.last_warning.elapsed() >= BACK_PRESSURE_WARN_INTERVAL {
stats.last_warning = Instant::now();
trc::event!(
Queue(trc::QueueEvent::BackPressure),
Reason = "Processing capacity for this queue exceeded.",
QueueName = queue_event.queue_name.to_string(),
Limit = stats.max_in_flight,
);
}
self.has_pending_work = true;
if queue_event.due < self.scan_from {
self.scan_from = queue_event.due;
}
}
}
// Remove expired locks, revisiting any event they were holding back
let scan_ceiling = self.scan_ceiling;
let mut dropped_due = u64::MAX;
self.locked.retain(|_, locked| {
let keep = locked.expires > now
&& (locked.revision == self.locked_revision
|| locked.due < scan_floor
|| locked.due >= scan_ceiling);
if !keep && locked.due < dropped_due {
dropped_due = locked.due;
}
keep
});
// Do not wait for the next scheduled event while there is work left over
let mut next_refresh = queue_events.next_refresh.saturating_sub(now);
if self.has_pending_work {
next_refresh = std::cmp::min(next_refresh, FULL_SCAN_INTERVAL.as_secs());
}
let mut next_refresh = Instant::now() + Duration::from_secs(next_refresh);
// A released lock uncovered an event below the floor that no scan can see
if dropped_due < self.scan_from {
self.scan_from = dropped_due;
self.has_pending_work = true;
self.pending_refresh = true;
let scan_at = self.last_scan + MIN_SCAN_INTERVAL;
if scan_at < next_refresh {
next_refresh = scan_at;
}
}
self.next_refresh = next_refresh;
}
}
async fn handle_event(&mut self, event: QueueEvent) -> bool {
match event {
QueueEvent::WorkerDone {
queue_id,
queue_name,
status,
} => {
let has_capacity = match self.stats.get_mut(&queue_name) {
Some(queue_stats) => {
queue_stats.in_flight = queue_stats.in_flight.saturating_sub(1);
queue_stats.has_capacity()
}
None => true,
};
match status {
QueueEventStatus::Completed => {
self.core.ipc.task_tx.notify_one();
self.locked.remove(&(queue_id, queue_name));
!self.locked.is_empty() || !has_capacity || self.has_pending_work
}
QueueEventStatus::Locked => {
let expires = LOCK_EXPIRY + rand::rng().random_range(5..10);
let due_in = Instant::now() + Duration::from_secs(expires);
if due_in < self.next_refresh {
self.next_refresh = due_in;
}
// The event was not delivered, so it has to be visited again
// once the remote lock expires.
let expires = now() + expires;
let due = match self.locked.entry((queue_id, queue_name)) {
Entry::Occupied(mut entry) => {
let locked = entry.get_mut();
locked.expires = expires;
locked.revision = self.locked_revision;
locked.due
}
Entry::Vacant(entry) => {
entry.insert(LockedMessage {
expires,
revision: self.locked_revision,
due: 0,
});
0
}
};
if due < self.scan_from {
self.scan_from = due;
}
self.locked.len() > 1 || !has_capacity || self.has_pending_work
}
QueueEventStatus::Deferred => {
self.locked.remove(&(queue_id, queue_name));
self.scan_from = 0;
true
}
}
}
QueueEvent::Refresh => {
self.scan_from = 0;
self.urgent_refresh = true;
true
}
QueueEvent::Paused(paused) => {
self.core
.data
.queue_status
.store(!paused, Ordering::Relaxed);
self.is_paused = paused;
self.scan_from = 0;
self.urgent_refresh = !paused;
!paused
}
QueueEvent::ReloadSettings => {
let server = self.core.build_server();
let virtual_queues = &server.core.smtp.queue.virtual_queues;
for (name, settings) in virtual_queues {
if let Some(stats) = self.stats.get_mut(name) {
stats.max_in_flight = settings.threads;
} else {
self.stats.insert(*name, QueueStats::new(settings.threads));
}
}
self.stats
.retain(|name, stats| stats.in_flight > 0 || virtual_queues.contains_key(name));
self.scan_from = 0;
false
}
QueueEvent::Stop => {
self.rx.close();
self.is_paused = true;
false
}
}
}
}
impl Message {
pub fn next_event(&self, queue: Option<QueueName>) -> Option<u64> {
let mut next_event = None;
for rcpt in &self.recipients {
if matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_))
&& queue.is_none_or(|q| rcpt.queue == q)
{
let mut earlier_event = std::cmp::min(rcpt.retry.due, rcpt.notify.due);
if let Some(expires) = rcpt.expiration_time(self.created) {
earlier_event = std::cmp::min(earlier_event, expires);
}
if let Some(next_event) = &mut next_event {
if earlier_event < *next_event {
*next_event = earlier_event;
}
} else {
next_event = Some(earlier_event);
}
}
}
next_event
}
pub fn next_delivery_event(&self, queue: Option<QueueName>) -> Option<u64> {
let mut next_delivery = None;
for rcpt in self.recipients.iter().filter(|rcpt| {
matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_))
&& queue.is_none_or(|q| rcpt.queue == q)
}) {
if let Some(next_delivery) = &mut next_delivery {
if rcpt.retry.due < *next_delivery {
*next_delivery = rcpt.retry.due;
}
} else {
next_delivery = Some(rcpt.retry.due);
}
}
next_delivery
}
pub fn next_dsn(&self, queue: Option<QueueName>) -> Option<u64> {
let mut next_dsn = None;
for rcpt in self.recipients.iter().filter(|rcpt| {
matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_))
&& queue.is_none_or(|q| rcpt.queue == q)
}) {
if let Some(next_dsn) = &mut next_dsn {
if rcpt.notify.due < *next_dsn {
*next_dsn = rcpt.notify.due;
}
} else {
next_dsn = Some(rcpt.notify.due);
}
}
next_dsn
}
pub fn expires(&self, queue: Option<QueueName>) -> Option<u64> {
let mut expires = None;
for rcpt in self.recipients.iter().filter(|d| {
matches!(d.status, Status::Scheduled | Status::TemporaryFailure(_))
&& queue.is_none_or(|q| d.queue == q)
}) {
if let Some(rcpt_expires) = rcpt.expiration_time(self.created) {
if let Some(expires) = &mut expires {
if rcpt_expires > *expires {
*expires = rcpt_expires;
}
} else {
expires = Some(rcpt_expires)
}
}
}
expires
}
pub fn next_events(&self) -> AHashMap<QueueName, u64> {
let mut next_events = AHashMap::new();
for rcpt in &self.recipients {
if matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) {
let mut earlier_event = std::cmp::min(rcpt.retry.due, rcpt.notify.due);
if let Some(expires) = rcpt.expiration_time(self.created) {
earlier_event = std::cmp::min(earlier_event, expires);
}
match next_events.entry(rcpt.queue) {
Entry::Occupied(mut entry) => {
let entry = entry.get_mut();
if earlier_event < *entry {
*entry = earlier_event;
}
}
Entry::Vacant(entry) => {
entry.insert(earlier_event);
}
}
}
}
next_events
}
}
impl Recipient {
pub fn expiration_time(&self, created: u64) -> Option<u64> {
match self.expires {
QueueExpiry::Ttl(time) => Some(created + time),
QueueExpiry::Attempts(_) => None,
}
}
pub fn is_expired(&self, created: u64, now: u64) -> bool {
match self.expires {
QueueExpiry::Ttl(time) => created + time <= now,
QueueExpiry::Attempts(count) => self.retry.inner >= count,
}
}
}
pub trait SpawnQueue {
fn spawn(self, core: Arc<Inner>);
}
impl QueueStats {
pub(crate) fn new(max_in_flight: usize) -> Self {
QueueStats {
in_flight: 0,
max_in_flight,
budget: 0,
last_warning: Instant::now()
.checked_sub(BACK_PRESSURE_WARN_INTERVAL)
.unwrap_or_else(Instant::now),
}
}
#[inline]
pub fn has_capacity(&self) -> bool {
self.in_flight < self.max_in_flight
}
}
crates/smtp/src/queue/mod.rs
+666
View File
@@ -0,0 +1,666 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::{
config::smtp::queue::{QueueExpiry, QueueName},
expr::{self, functions::ResolveVariable, *},
};
use compact_str::ToCompactString;
use registry::schema::enums::ExpressionVariable;
use smtp_proto::Response;
use std::{
fmt::Display,
net::{IpAddr, Ipv4Addr},
time::{Duration, Instant, SystemTime},
};
use store::write::now;
use types::blob_hash::BlobHash;
use utils::DomainPart;
pub mod dsn;
pub mod manager;
pub mod quota;
pub mod spool;
pub mod throttle;
pub type QueueId = u64;
#[derive(Debug, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize)]
pub struct Schedule<T> {
pub due: u64,
pub inner: T,
}
#[derive(Debug, Clone, Copy)]
pub struct QueuedMessage {
pub due: u64,
pub queue_id: QueueId,
pub queue_name: QueueName,
}
#[derive(Debug, Clone, Copy)]
pub enum MessageSource {
Authenticated,
Unauthenticated { dmarc_pass: bool },
Dsn,
Report,
Autogenerated,
}
impl MessageSource {
pub fn flags(&self) -> u64 {
match self {
MessageSource::Authenticated => FROM_AUTHENTICATED,
MessageSource::Unauthenticated { dmarc_pass: true } => FROM_UNAUTHENTICATED_DMARC,
MessageSource::Unauthenticated { dmarc_pass: false } => FROM_UNAUTHENTICATED,
MessageSource::Dsn => FROM_DSN,
MessageSource::Report => FROM_REPORT,
MessageSource::Autogenerated => FROM_AUTOGENERATED,
}
}
}
#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq)]
pub struct Message {
pub created: u64,
pub blob_hash: BlobHash,
pub return_path: Box<str>,
pub recipients: Vec<Recipient>,
pub received_from_ip: IpAddr,
pub received_via_port: u16,
pub flags: u64,
pub env_id: Option<Box<str>>,
pub priority: i16,
pub size: u64,
pub metadata: Box<[Metadata]>,
}
impl Message {
pub fn queued_event(&self) -> trc::QueueEvent {
if (self.flags & FROM_AUTHENTICATED) != 0 {
trc::QueueEvent::AuthenticatedMessageQueued
} else if (self.flags & FROM_DSN) != 0 {
trc::QueueEvent::DsnQueued
} else if (self.flags & FROM_REPORT) != 0 {
trc::QueueEvent::ReportQueued
} else if (self.flags & FROM_AUTOGENERATED) != 0 {
trc::QueueEvent::AutogeneratedQueued
} else {
trc::QueueEvent::MessageQueued
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MessageWrapper {
pub queue_id: QueueId,
pub queue_name: QueueName,
pub is_multi_queue: bool,
pub span_id: u64,
pub message: Message,
}
#[derive(
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
Debug,
Clone,
PartialEq,
Eq,
serde::Deserialize,
)]
pub enum Metadata {
QueueSize { key: Box<[u8]>, id: u64 },
QueueCount { key: Box<[u8]>, id: u64 },
Headers { value: Box<[u8]>, id: u64 },
}
#[derive(
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
Debug,
Clone,
PartialEq,
Eq,
serde::Deserialize,
)]
pub struct Recipient {
pub address: Box<str>,
pub retry: Schedule<u32>,
pub notify: Schedule<u32>,
pub expires: QueueExpiry,
pub queue: QueueName,
pub status: Status<HostResponse<Box<str>>, ErrorDetails>,
pub flags: u64,
pub orcpt: Option<Box<str>>,
}
pub const FROM_AUTHENTICATED: u64 = 1 << 32;
pub const FROM_UNAUTHENTICATED: u64 = 1 << 33;
pub const FROM_UNAUTHENTICATED_DMARC: u64 = 1 << 34;
pub const FROM_DSN: u64 = 1 << 35;
pub const FROM_REPORT: u64 = 1 << 36;
pub const FROM_AUTOGENERATED: u64 = 1 << 37;
pub const RCPT_DSN_SENT: u64 = 1 << 32;
pub const RCPT_SPAM_SHIFT: u64 = 56;
pub const RCPT_SPAM_MASK: u64 = 0xff << RCPT_SPAM_SHIFT;
pub const fn rcpt_spam_flag(percentage: u8) -> u64 {
(percentage as u64 + 1) << RCPT_SPAM_SHIFT
}
pub const fn rcpt_spam_percentage(flags: u64) -> Option<u8> {
match (flags >> RCPT_SPAM_SHIFT) as u8 {
0 => None,
percentage => Some(percentage - 1),
}
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
serde::Serialize,
serde::Deserialize,
)]
pub enum Status<T, E> {
#[serde(rename = "scheduled")]
Scheduled,
#[serde(rename = "completed")]
Completed(T),
#[serde(rename = "temp_fail")]
TemporaryFailure(E),
#[serde(rename = "perm_fail")]
PermanentFailure(E),
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
serde::Deserialize,
)]
pub struct HostResponse<T> {
pub hostname: T,
pub response: Response<Box<str>>,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
serde::Deserialize,
Default,
)]
pub enum Error {
DnsError(Box<str>),
UnexpectedResponse(UnexpectedResponse),
ConnectionError(Box<str>),
TlsError(Box<str>),
DaneError(Box<str>),
MtaStsError(Box<str>),
RateLimited,
#[default]
ConcurrencyLimited,
Io(Box<str>),
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
serde::Deserialize,
)]
pub struct UnexpectedResponse {
pub command: Box<str>,
pub response: Response<Box<str>>,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
rkyv::Serialize,
rkyv::Deserialize,
rkyv::Archive,
Default,
serde::Deserialize,
)]
pub struct ErrorDetails {
pub entity: Box<str>,
pub details: Error,
}
impl<T> Ord for Schedule<T> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
other.due.cmp(&self.due)
}
}
impl<T> PartialOrd for Schedule<T> {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl<T> PartialEq for Schedule<T> {
fn eq(&self, other: &Self) -> bool {
self.due == other.due
}
}
impl<T> Eq for Schedule<T> {}
impl<T: Default> Schedule<T> {
pub fn now() -> Self {
Schedule {
due: now(),
inner: T::default(),
}
}
pub fn later(duration: u64) -> Self {
Schedule {
due: now() + duration,
inner: T::default(),
}
}
}
pub struct QueueEnvelope<'x> {
pub message: &'x Message,
pub domain: &'x str,
pub mx: &'x str,
pub rcpt: &'x Recipient,
pub remote_ip: IpAddr,
pub local_ip: IpAddr,
}
impl<'x> QueueEnvelope<'x> {
pub fn new(message: &'x Message, rcpt: &'x Recipient) -> Self {
Self {
message,
domain: rcpt.address.domain_part(),
rcpt,
mx: "",
remote_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
local_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
}
}
}
impl<'x> ResolveVariable for QueueEnvelope<'x> {
fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'x> {
match variable {
ExpressionVariable::Sender => self.message.return_path.as_ref().into(),
ExpressionVariable::SenderDomain => self.message.return_path.domain_part().into(),
ExpressionVariable::RcptDomain => self.domain.into(),
ExpressionVariable::Rcpt => self.rcpt.address.as_ref().into(),
ExpressionVariable::Recipients => self
.message
.recipients
.iter()
.map(|r| Variable::from(r.address.as_ref()))
.collect::<Vec<_>>()
.into(),
ExpressionVariable::RetryNum => self.rcpt.retry.inner.into(),
ExpressionVariable::NotifyNum => self.rcpt.notify.inner.into(),
ExpressionVariable::ExpiresIn => match &self.rcpt.expires {
QueueExpiry::Ttl(time) => (*time + self.message.created).saturating_sub(now()),
QueueExpiry::Attempts(count) => {
(count.saturating_sub(self.rcpt.retry.inner)) as u64
}
}
.into(),
ExpressionVariable::LastStatus => self.rcpt.status.to_compact_string().into(),
ExpressionVariable::LastError => match &self.rcpt.status {
Status::Scheduled | Status::Completed(_) => "none",
Status::TemporaryFailure(err) | Status::PermanentFailure(err) => {
match &err.details {
Error::DnsError(_) => "dns",
Error::UnexpectedResponse(_) => "unexpected-reply",
Error::ConnectionError(_) => "connection",
Error::TlsError(_) => "tls",
Error::DaneError(_) => "dane",
Error::MtaStsError(_) => "mta-sts",
Error::RateLimited => "rate",
Error::ConcurrencyLimited => "concurrency",
Error::Io(_) => "io",
}
}
}
.into(),
ExpressionVariable::QueueName => self.rcpt.queue.as_str().into(),
ExpressionVariable::QueueAge => now().saturating_sub(self.message.created).into(),
ExpressionVariable::Source => if (self.message.flags & FROM_AUTHENTICATED) != 0 {
"authenticated"
} else if (self.message.flags & FROM_UNAUTHENTICATED_DMARC) != 0 {
"dmarc_pass"
} else if (self.message.flags & FROM_UNAUTHENTICATED) != 0 {
"unauthenticated"
} else if (self.message.flags & FROM_DSN) != 0 {
"dsn"
} else if (self.message.flags & FROM_REPORT) != 0 {
"report"
} else if (self.message.flags & FROM_AUTOGENERATED) != 0 {
"autogenerated"
} else {
"unknown"
}
.into(),
ExpressionVariable::Mx => self.mx.into(),
ExpressionVariable::Priority => self.message.priority.into(),
ExpressionVariable::RemoteIp => self.remote_ip.to_compact_string().into(),
ExpressionVariable::LocalIp => self.local_ip.to_compact_string().into(),
ExpressionVariable::ReceivedFromIp => {
self.message.received_from_ip.to_compact_string().into()
}
ExpressionVariable::ReceivedViaPort => self.message.received_via_port.into(),
ExpressionVariable::Size => self.message.size.into(),
_ => "".into(),
}
}
fn resolve_global(&self, _: &str) -> Variable<'_> {
Variable::Integer(0)
}
}
impl ResolveVariable for Message {
fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'_> {
match variable {
ExpressionVariable::Sender => self.return_path.as_ref().into(),
ExpressionVariable::SenderDomain => self.return_path.domain_part().into(),
ExpressionVariable::Recipients => self
.recipients
.iter()
.map(|r| Variable::from(r.address.as_ref()))
.collect::<Vec<_>>()
.into(),
ExpressionVariable::Priority => self.priority.into(),
_ => "".into(),
}
}
fn resolve_global(&self, _: &str) -> Variable<'_> {
Variable::Integer(0)
}
}
impl ResolveVariable for MessageWrapper {
fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'_> {
match variable {
ExpressionVariable::Sender => self.message.return_path.as_ref().into(),
ExpressionVariable::SenderDomain => self.message.return_path.domain_part().into(),
ExpressionVariable::Recipients => self
.message
.recipients
.iter()
.map(|r| Variable::from(r.address.as_ref()))
.collect::<Vec<_>>()
.into(),
ExpressionVariable::Priority => self.message.priority.into(),
ExpressionVariable::QueueName => self.queue_name.as_str().into(),
ExpressionVariable::QueueAge => now().saturating_sub(self.message.created).into(),
ExpressionVariable::Source => if (self.message.flags & FROM_AUTHENTICATED) != 0 {
"authenticated"
} else if (self.message.flags & FROM_UNAUTHENTICATED_DMARC) != 0 {
"dmarc_pass"
} else if (self.message.flags & FROM_UNAUTHENTICATED) != 0 {
"unauthenticated"
} else if (self.message.flags & FROM_DSN) != 0 {
"dsn"
} else if (self.message.flags & FROM_REPORT) != 0 {
"report"
} else if (self.message.flags & FROM_AUTOGENERATED) != 0 {
"autogenerated"
} else {
"unknown"
}
.into(),
ExpressionVariable::ReceivedFromIp => {
self.message.received_from_ip.to_compact_string().into()
}
ExpressionVariable::ReceivedViaPort => self.message.received_via_port.into(),
ExpressionVariable::Size => self.message.size.into(),
_ => "".into(),
}
}
fn resolve_global(&self, _: &str) -> Variable<'_> {
Variable::Integer(0)
}
}
pub struct RecipientDomain<'x>(&'x str);
impl<'x> RecipientDomain<'x> {
pub fn new(domain: &'x str) -> Self {
Self(domain)
}
}
impl<'x> ResolveVariable for RecipientDomain<'x> {
fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'x> {
match variable {
ExpressionVariable::RcptDomain => self.0.into(),
_ => "".into(),
}
}
fn resolve_global(&self, _: &str) -> Variable<'_> {
Variable::Integer(0)
}
}
#[inline(always)]
pub fn instant_to_timestamp(now: Instant, time: Instant) -> u64 {
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(0, |d| d.as_secs())
+ time.checked_duration_since(now).map_or(0, |d| d.as_secs())
}
impl Recipient {
pub fn new(address: impl AsRef<str>) -> Self {
Recipient {
address: address.to_lowercase_address(false).into_boxed_str(),
status: Status::Scheduled,
flags: 0,
orcpt: None,
retry: Schedule::now(),
notify: Schedule::now(),
expires: QueueExpiry::Attempts(0),
queue: QueueName::default(),
}
}
pub fn with_flags(mut self, flags: u64) -> Self {
self.flags = flags;
self
}
pub fn with_orcpt(mut self, orcpt: Option<Box<str>>) -> Self {
self.orcpt = orcpt;
self
}
pub fn address(&self) -> &str {
&self.address
}
pub fn domain_part(&self) -> &str {
self.address.domain_part()
}
}
impl ArchivedRecipient {
pub fn address(&self) -> &str {
self.address.as_ref()
}
pub fn domain_part(&self) -> &str {
self.address.domain_part()
}
}
pub trait InstantFromTimestamp {
fn to_instant(&self) -> Instant;
}
impl InstantFromTimestamp for u64 {
fn to_instant(&self) -> Instant {
let timestamp = *self;
let current_timestamp = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
if timestamp > current_timestamp {
Instant::now() + Duration::from_secs(timestamp - current_timestamp)
} else {
Instant::now()
}
}
}
impl Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::UnexpectedResponse(response) => {
write!(
f,
"Unexpected response for {}: {}",
response.command, response.response
)
}
Error::DnsError(err) => {
write!(f, "DNS lookup failed: {err}")
}
Error::ConnectionError(details) => {
write!(f, "Connection failed: {details}",)
}
Error::TlsError(details) => {
write!(f, "TLS error: {details}",)
}
Error::DaneError(details) => {
write!(f, "DANE authentication failure: {details}",)
}
Error::MtaStsError(details) => {
write!(f, "MTA-STS auth failed: {details}")
}
Error::RateLimited => {
write!(f, "Rate limited")
}
Error::ConcurrencyLimited => {
write!(f, "Too many concurrent connections to remote server")
}
Error::Io(err) => {
write!(f, "Queue error: {err}")
}
}
}
}
impl Display for ArchivedError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ArchivedError::UnexpectedResponse(response) => {
write!(
f,
"Unexpected response for {}: {}",
response.command, response.response
)
}
ArchivedError::DnsError(err) => {
write!(f, "DNS lookup failed: {err}")
}
ArchivedError::ConnectionError(details) => {
write!(f, "Connection failed: {details}",)
}
ArchivedError::TlsError(details) => {
write!(f, "TLS error: {details}",)
}
ArchivedError::DaneError(details) => {
write!(f, "DANE authentication failure: {details}",)
}
ArchivedError::MtaStsError(details) => {
write!(f, "MTA-STS auth failed: {details}")
}
ArchivedError::RateLimited => {
write!(f, "Rate limited")
}
ArchivedError::ConcurrencyLimited => {
write!(f, "Too many concurrent connections to remote server")
}
ArchivedError::Io(err) => {
write!(f, "Queue error: {err}")
}
}
}
}
impl Display for Status<HostResponse<Box<str>>, ErrorDetails> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Status::Scheduled => write!(f, "Scheduled"),
Status::Completed(response) => write!(f, "Delivered: {}", response.response),
Status::TemporaryFailure(err) => {
write!(f, "Temporary Failure for {}: {}", err.entity, err.details)
}
Status::PermanentFailure(err) => {
write!(f, "Permanent Failure for {}: {}", err.entity, err.details)
}
}
}
}
impl Display for ArchivedErrorDetails {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Error for {}: {}", self.entity, self.details)
}
}
/*
pub trait DisplayArchivedResponse {
fn to_string(&self) -> String;
}
impl DisplayArchivedResponse for ArchivedResponse<Box<str>> {
fn to_string(&self) -> String {
format!(
"Code: {}, Enhanced code: {}.{}.{}, Message: {}",
self.code, self.esc[0], self.esc[1], self.esc[2], self.message,
)
}
}
*/
crates/smtp/src/queue/quota.rs
+230
View File
@@ -0,0 +1,230 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::{Metadata, QueueEnvelope, Status};
use crate::{core::throttle::NewKey, queue::MessageWrapper};
use ahash::AHashSet;
use common::{Server, config::smtp::queue::QueueQuota, expr::functions::ResolveVariable};
use std::future::Future;
use store::{
ValueKey,
write::{BatchBuilder, QueueClass, ValueClass},
};
use trc::QueueEvent;
use utils::DomainPart;
pub trait HasQueueQuota: Sync + Send {
fn has_quota(
&self,
message: &mut MessageWrapper,
) -> impl Future<Output = Option<Vec<Metadata>>> + Send;
fn check_quota<'x>(
&'x self,
quota: &'x QueueQuota,
envelope: &impl ResolveVariable,
size: u64,
id: u64,
refs: &mut Vec<Metadata>,
session_id: u64,
) -> impl Future<Output = bool> + Send;
}
impl HasQueueQuota for Server {
async fn has_quota(&self, message: &mut MessageWrapper) -> Option<Vec<Metadata>> {
let mut quota_keys = Vec::new();
if !self.core.smtp.queue.quota.sender.is_empty() {
for quota in &self.core.smtp.queue.quota.sender {
if !self
.check_quota(
quota,
&message.message,
message.message.size,
0,
&mut quota_keys,
message.span_id,
)
.await
{
trc::event!(
Queue(QueueEvent::QuotaExceeded),
SpanId = message.span_id,
Id = quota.id.to_string(),
Type = "Sender"
);
return None;
}
}
}
if !self.core.smtp.queue.quota.rcpt_domain.is_empty() {
let mut seen_domains = AHashSet::new();
for quota in &self.core.smtp.queue.quota.rcpt_domain {
for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() {
if seen_domains.insert(rcpt.address.domain_part())
&& !self
.check_quota(
quota,
&QueueEnvelope::new(&message.message, rcpt),
message.message.size,
((rcpt_idx + 1) << 32) as u64,
&mut quota_keys,
message.span_id,
)
.await
{
trc::event!(
Queue(QueueEvent::QuotaExceeded),
SpanId = message.span_id,
Id = quota.id.to_string(),
Type = "Domain"
);
return None;
}
}
}
}
for quota in &self.core.smtp.queue.quota.rcpt {
for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() {
if !self
.check_quota(
quota,
&QueueEnvelope::new(&message.message, rcpt),
message.message.size,
(rcpt_idx + 1) as u64,
&mut quota_keys,
message.span_id,
)
.await
{
trc::event!(
Queue(QueueEvent::QuotaExceeded),
SpanId = message.span_id,
Id = quota.id.to_string(),
Type = "Recipient"
);
return None;
}
}
}
Some(quota_keys)
}
async fn check_quota<'x>(
&'x self,
quota: &'x QueueQuota,
envelope: &impl ResolveVariable,
size: u64,
id: u64,
refs: &mut Vec<Metadata>,
session_id: u64,
) -> bool {
if !quota.expr.is_empty()
&& self
.eval_if(&quota.expr, envelope, session_id)
.await
.unwrap_or(false)
{
let key = quota.new_key(envelope, "");
if let Some(max_size) = quota.size {
let used_size = self
.core
.storage
.data
.get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaSize(
key.as_ref().to_vec(),
))))
.await
.unwrap_or(0) as u64;
if used_size + size > max_size {
return false;
} else {
refs.push(Metadata::QueueSize {
key: key.as_ref().into(),
id,
});
}
}
if let Some(max_messages) = quota.messages {
let total_messages = self
.core
.storage
.data
.get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaCount(
key.as_ref().to_vec(),
))))
.await
.unwrap_or(0) as u64;
if total_messages + 1 > max_messages {
return false;
} else {
refs.push(Metadata::QueueCount {
key: key.as_ref().into(),
id,
});
}
}
}
true
}
}
impl MessageWrapper {
pub fn release_quota(&mut self, batch: &mut BatchBuilder) {
if !self.message.metadata.iter().any(|metadata| {
matches!(
metadata,
Metadata::QueueSize { .. } | Metadata::QueueCount { .. }
)
}) {
return;
}
let mut quota_ids = Vec::with_capacity(self.message.recipients.len());
let mut seen_domains = AHashSet::new();
for (pos, rcpt) in self.message.recipients.iter().enumerate() {
if matches!(
&rcpt.status,
Status::Completed(_) | Status::PermanentFailure(_)
) {
if seen_domains.insert(rcpt.address.domain_part()) {
quota_ids.push(((pos + 1) as u64) << 32);
}
quota_ids.push((pos + 1) as u64);
}
}
if !quota_ids.is_empty() {
let mut metadata = Vec::new();
for entry in std::mem::take(&mut self.message.metadata) {
match entry {
Metadata::QueueCount { id, key } if quota_ids.contains(&id) => {
batch.add(
ValueClass::Queue(QueueClass::QuotaCount(key.into_vec())),
-1,
);
}
Metadata::QueueSize { id, key } if quota_ids.contains(&id) => {
batch.add(
ValueClass::Queue(QueueClass::QuotaSize(key.into_vec())),
-(self.message.size as i64),
);
}
_ => {
metadata.push(entry);
}
}
}
self.message.metadata = metadata.into_boxed_slice();
}
}
}
crates/smtp/src/queue/spool.rs
+1152
View File
File diff suppressed because it is too large Load Diff
crates/smtp/src/queue/throttle.rs
+65
View File
@@ -0,0 +1,65 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::core::throttle::NewKey;
use common::{
KV_RATE_LIMIT_SMTP, Server, config::smtp::QueueRateLimiter, expr::functions::ResolveVariable,
};
use std::future::Future;
use store::write::now;
pub trait IsAllowed: Sync + Send {
fn is_allowed<'x>(
&'x self,
throttle: &'x QueueRateLimiter,
envelope: &impl ResolveVariable,
session_id: u64,
) -> impl Future<Output = Result<(), u64>> + Send;
}
impl IsAllowed for Server {
async fn is_allowed<'x>(
&'x self,
throttle: &'x QueueRateLimiter,
envelope: &impl ResolveVariable,
session_id: u64,
) -> Result<(), u64> {
if throttle.expr.is_empty()
|| self
.eval_if(&throttle.expr, envelope, session_id)
.await
.unwrap_or(false)
{
let key = throttle.new_key(envelope, "outbound");
match self
.in_memory_store()
.is_rate_allowed(KV_RATE_LIMIT_SMTP, key.as_ref(), &throttle.rate, false)
.await
{
Ok(Some(next_refill)) => {
trc::event!(
Queue(trc::QueueEvent::RateLimitExceeded),
SpanId = session_id,
Id = throttle.id.to_string(),
Limit = vec![
trc::Value::from(throttle.rate.count),
trc::Value::from(throttle.rate.period.into_inner())
],
);
return Err(now() + next_refill);
}
Err(err) => {
trc::error!(err.span_id(session_id).caused_by(trc::location!()));
}
_ => (),
}
}
Ok(())
}
}