Import upstream v0.16.23, stripped

Upstream commit: 9d1c75ab68435e4417337f768291e5f947686203
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Verification: clean

One snippet more than v0.16.22, in crates/common/src/auth/authentication.rs
(3, was 2).

crates/smtp/src/queue/dsn.rs

@@ -10,9 +10,10 @@ use super::{
     Recipient, Status,
 };
 use crate::inbound::dkim::DkimSign;
-use crate::queue::spool::QueueParams;
+use crate::queue::spool::{DSN_RETRY, QueueParams};
 use crate::queue::{MessageWrapper, UnexpectedResponse};
 use common::Server;
+use email::message::delivery::ORCPT_ADDR_TYPE;
 use mail_builder::MessageBuilder;
 use mail_builder::headers::HeaderType;
 use mail_builder::headers::content_type::ContentType;
@@ -25,16 +26,24 @@ use std::fmt::Write;
 use std::future::Future;
 use store::write::now;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DsnStatus {
+    Completed,
+    Deferred,
+}
+
 pub trait SendDsn: Sync + Send {
-    fn send_dsn(&self, message: &mut MessageWrapper) -> impl Future<Output = ()> + Send;
+    fn send_dsn(&self, message: &mut MessageWrapper) -> impl Future<Output = DsnStatus> + Send;
     fn log_dsn(&self, message: &MessageWrapper) -> impl Future<Output = ()> + Send;
 }
 
 impl SendDsn for Server {
-    async fn send_dsn(&self, message: &mut MessageWrapper) {
+    async fn send_dsn(&self, message: &mut MessageWrapper) -> DsnStatus {
         // Send DSN events
         self.log_dsn(message).await;
 
+        let mut status = DsnStatus::Completed;
+
         if !message.message.return_path.is_empty() {
             // Build DSN
             if let Some(dsn) = message.build_dsn(self).await {
@@ -51,12 +60,19 @@ impl SendDsn for Server {
                         message.span_id,
                     )
                     .await;
-                dsn_message
+                if dsn_message
                     .queue(
                         QueueParams::new(&dsn, message.span_id, self)
                             .with_dkim_signers(dkim_signers),
                     )
-                    .await;
+                    .await
+                {
+                    message.mark_dsn_sent();
+                } else {
+                    status = DsnStatus::Deferred;
+                }
+            } else {
+                message.mark_dsn_sent();
             }
         } else {
             // Handle double bounce
@@ -64,7 +80,9 @@ impl SendDsn for Server {
         }
 
         // Update next DSN notify times
-        message.update_next_dsn(self).await;
+        message.update_next_dsn(self, status).await;
+
+        status
     }
 
     async fn log_dsn(&self, message: &MessageWrapper) {
@@ -132,7 +150,7 @@ impl SendDsn for Server {
 const MAX_HEADER_SIZE: usize = 4096;
 
 impl MessageWrapper {
-    pub async fn build_dsn(&mut self, server: &Server) -> Option<Vec<u8>> {
+    pub async fn build_dsn(&self, server: &Server) -> Option<Vec<u8>> {
         let config = &server.core.smtp.queue;
         let now = now();
 
@@ -141,13 +159,12 @@ impl MessageWrapper {
         let mut txt_failed = String::new();
         let mut dsn = String::new();
 
-        for rcpt in &mut self.message.recipients {
+        for rcpt in &self.message.recipients {
             if rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) {
                 continue;
             }
             match &rcpt.status {
                 Status::Completed(response) => {
-                    rcpt.flags |= RCPT_DSN_SENT;
                     if !rcpt.has_flag(RCPT_NOTIFY_SUCCESS) {
                         continue;
                     }
@@ -164,7 +181,6 @@ impl MessageWrapper {
                     response.write_dsn_text(&rcpt.address, &mut txt_delay);
                 }
                 Status::PermanentFailure(response) => {
-                    rcpt.flags |= RCPT_DSN_SENT;
                     if !rcpt.has_flag(RCPT_NOTIFY_FAILURE) {
                         continue;
                     }
@@ -357,7 +373,7 @@ impl MessageWrapper {
             .into()
     }
 
-    pub async fn update_next_dsn(&mut self, server: &Server) {
+    pub async fn update_next_dsn(&mut self, server: &Server, status: DsnStatus) {
         let now = now();
         let mut notify_changes = Vec::new();
         for (rcpt_idx, rcpt) in self.message.recipients.iter().enumerate() {
@@ -366,6 +382,11 @@ impl MessageWrapper {
                 Status::TemporaryFailure(_) | Status::Scheduled
             ) && rcpt.notify.due <= now
             {
+                if status == DsnStatus::Deferred {
+                    notify_changes.push((rcpt_idx, 0, now + DSN_RETRY));
+                    continue;
+                }
+
                 let envelope = QueueEnvelope::new(&self.message, rcpt);
 
                 let queue_id = server
@@ -391,6 +412,19 @@ impl MessageWrapper {
         }
     }
 
+    fn mark_dsn_sent(&mut self) {
+        for rcpt in &mut self.message.recipients {
+            if !rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER)
+                && matches!(
+                    rcpt.status,
+                    Status::Completed(_) | Status::PermanentFailure(_)
+                )
+            {
+                rcpt.flags |= RCPT_DSN_SENT;
+            }
+        }
+    }
+
     fn handle_double_bounce(&mut self) {
         let mut is_double_bounce = Vec::with_capacity(0);
         let now = now();
@@ -523,7 +557,7 @@ impl Message {
 impl Recipient {
     fn write_dsn(&self, dsn: &mut String) {
         if let Some(orcpt) = &self.orcpt {
-            let _ = write!(dsn, "Original-Recipient: rfc822;{orcpt}\r\n");
+            let _ = write!(dsn, "Original-Recipient: {ORCPT_ADDR_TYPE}{orcpt}\r\n");
         }
         let _ = write!(dsn, "Final-Recipient: rfc822;{}\r\n", self.address);
     }

crates/smtp/src/queue/spool.rs

@@ -45,6 +45,7 @@ use utils::DomainPart;
 
 pub const LOCK_EXPIRY: u64 = 10 * 60; // 10 minutes
 pub const QUEUE_REFRESH: u64 = 5 * 60; // 5 minutes
+pub const DSN_RETRY: u64 = 5 * 60; // 5 minutes
 pub(crate) const INFINITE_LOCK: u64 = 60 * 60 * 24 * 365; // 1 year
 const CANDIDATE_OVERSCAN: usize = 4;
 const MAX_PREALLOCATED_CANDIDATES: usize = 1024;
@@ -370,6 +371,7 @@ pub(crate) struct QueueParams<'x, 'y> {
 }
 
 impl MessageWrapper {
+    #[must_use]
     pub(crate) async fn queue<'x, 'y>(mut self, mut params: QueueParams<'x, 'y>) -> bool {
         // Add DKIM signatures
         let dkim_headers = if params.dkim_signers.is_some() {
@@ -669,7 +671,12 @@ impl MessageWrapper {
         recipient.queue = queue.virtual_queue;
     }
 
-    pub async fn save_changes(mut self, server: &Server, prev_event: Option<u64>) -> bool {
+    pub async fn save_changes(
+        mut self,
+        server: &Server,
+        prev_event: Option<u64>,
+        retry_at: Option<u64>,
+    ) -> bool {
         // Release quota for completed deliveries
         let mut batch = BatchBuilder::new();
         self.release_quota(&mut batch);
@@ -684,7 +691,12 @@ impl MessageWrapper {
                 },
             )));
         }
-        for (queue_name, due) in self.message.next_events() {
+        let mut next_events = self.message.next_events();
+        if let Some(retry_at) = retry_at {
+            let due = next_events.entry(self.queue_name).or_insert(retry_at);
+            *due = std::cmp::min(*due, retry_at);
+        }
+        for (queue_name, due) in next_events {
             batch.set(
                 ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent {
                     due,