Import upstream v0.16.23, stripped

Upstream commit: 9d1c75ab68435e4417337f768291e5f947686203
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Verification: clean

One snippet more than v0.16.22, in crates/common/src/auth/authentication.rs
(3, was 2).

crates/common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "common"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 build = "build.rs"
 

crates/common/src/auth/authentication.rs

@@ -9,7 +9,7 @@ use crate::{
     auth::{
         AccessToken, AuthRequest, DomainCache,
         credential::{ApiKey, AppPassword},
-        oauth::GrantType,
+        oauth::{GrantType, token::TOKEN_HEADER},
     },
 };
 use base64::{Engine, engine::general_purpose};
@@ -21,7 +21,8 @@ use registry::schema::{
     enums::Permission,
     structs::{self, Credential},
 };
-use std::{net::IpAddr, sync::Arc};
+use serde::Deserialize;
+use std::{borrow::Cow, net::IpAddr, sync::Arc};
 use store::write::now;
 use trc::AddContext;
 
@@ -319,19 +320,12 @@ impl Server {
                 // Obtain external directory, if any. When no username is supplied
                 // (e.g. HTTP bearer auth), peek at the JWT claims to find the
                 // user's domain so per-domain OIDC directories are reachable.
-                let directory = if let Some(username) = username.as_deref().map(UsernameParts::new)
-                {
-                    if let Some(domain_name) = username.auth_as().domain() {
-                        self.get_directory_for_domain(domain_name).await?
-                    } else if let Some(domain_name) = extract_jwt_domain(token) {
-                        self.get_directory_for_domain(&domain_name).await?
-                    } else {
-                        self.get_default_directory()
-                    }
-                } else if let Some(domain_name) = extract_jwt_domain(token) {
-                    self.get_directory_for_domain(&domain_name).await?
-                } else {
-                    self.get_default_directory()
+                let directory = match username.as_deref().map(UsernameParts::new) {
+                    Some(username) => match username.auth_as().domain() {
+                        Some(domain_name) => self.get_directory_for_domain(domain_name).await?,
+                        None => self.get_directory_for_token(token).await?,
+                    },
+                    None => self.get_directory_for_token(token).await?,
                 };
 
                 // Try external directory authentication first if supported, then fallback to internal OAuth.
@@ -520,31 +514,78 @@ impl Server {
         Ok(self.get_default_directory())
     }
 
+    async fn get_directory_for_token(&self, token: &str) -> trc::Result<Option<&Arc<Directory>>> {
+        let Some(payload) = JwtClaims::decode_payload(token) else {
+            return Ok(self.get_default_directory());
+        };
+        let Some(claims) = JwtClaims::parse(&payload) else {
+            return Ok(self.get_default_directory());
+        };
+
+        match (claims.domain(), claims.iss.as_deref()) {
+            (Some(domain_name), _) => self.get_directory_for_domain(domain_name).await,
+            (None, Some(issuer)) => Ok(self
+                .get_directory_for_issuer(issuer)
+                .or_else(|| self.get_default_directory())),
+            (None, None) => Ok(self.get_default_directory()),
+        }
+    }
+
+    fn get_directory_for_issuer(&self, issuer: &str) -> Option<&Arc<Directory>> {
+
+        None
+    }
+
     pub fn get_directory_for_cached_domain(&self, domain: &DomainCache) -> Option<&Arc<Directory>> {
 
         self.get_default_directory()
     }
 }
 
-fn extract_jwt_domain(token: &str) -> Option<String> {
-    let mut parts = token.split('.');
-    let _header = parts.next()?;
-    let payload = parts.next()?;
-    let _signature = parts.next()?;
-    if parts.next().is_some() {
-        return None;
-    }
-    let payload_bytes = general_purpose::URL_SAFE_NO_PAD.decode(payload).ok()?;
-    let claims: serde_json::Value = serde_json::from_slice(&payload_bytes).ok()?;
-    for claim in ["email", "preferred_username", "upn"] {
-        if let Some(val) = claims.get(claim).and_then(|v| v.as_str())
-            && let Some((_, domain)) = val.rsplit_once('@')
-            && !domain.is_empty()
-        {
-            return Some(domain.to_ascii_lowercase());
+#[derive(Deserialize)]
+struct JwtClaims<'x> {
+    #[serde(borrow, default)]
+    iss: Option<Cow<'x, str>>,
+    #[serde(borrow, default)]
+    email: Option<Cow<'x, str>>,
+    #[serde(borrow, default)]
+    preferred_username: Option<Cow<'x, str>>,
+    #[serde(borrow, default)]
+    upn: Option<Cow<'x, str>>,
+}
+
+impl<'x> JwtClaims<'x> {
+    fn decode_payload(token: &str) -> Option<Vec<u8>> {
+        if token.starts_with(TOKEN_HEADER) {
+            return None;
         }
+
+        let mut parts = token.split('.');
+        let _header = parts.next()?;
+        let payload = parts.next()?;
+        let _signature = parts.next()?;
+        if parts.next().is_some() {
+            return None;
+        }
+
+        general_purpose::URL_SAFE_NO_PAD.decode(payload).ok()
+    }
+
+    fn parse(payload: &'x [u8]) -> Option<Self> {
+        serde_json::from_slice(payload).ok()
+    }
+
+    fn domain(&self) -> Option<&str> {
+        [&self.email, &self.preferred_username, &self.upn]
+            .into_iter()
+            .flatten()
+            .find_map(|claim| {
+                claim
+                    .rsplit_once('@')
+                    .map(|(_, domain)| domain)
+                    .filter(|domain| !domain.is_empty())
+            })
     }
-    None
 }
 
 impl UsernameParts {
@@ -642,3 +683,76 @@ impl AuthRequest {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn jwt(payload: &str) -> String {
+        format!(
+            "eyJhbGciOiJSUzI1NiJ9.{}.c2lnbmF0dXJl",
+            general_purpose::URL_SAFE_NO_PAD.encode(payload)
+        )
+    }
+
+    fn hints(token: &str) -> Option<(Option<String>, Option<String>)> {
+        let payload = JwtClaims::decode_payload(token)?;
+        let claims = JwtClaims::parse(&payload)?;
+
+        Some((
+            claims.domain().map(str::to_string),
+            claims.iss.as_deref().map(str::to_string),
+        ))
+    }
+
+    #[test]
+    fn jwt_claims_are_extracted() {
+        for (payload, domain, issuer) in [
+            (
+                r#"{"iss":"https://idp.example.org","email":"[email protected]"}"#,
+                Some("Example.ORG"),
+                Some("https://idp.example.org"),
+            ),
+            (
+                r#"{"preferred_username":"[email protected]","upn":"[email protected]"}"#,
+                Some("example.net"),
+                None,
+            ),
+            (
+                r#"{"email":"broken@","upn":"[email protected]"}"#,
+                Some("example.com"),
+                None,
+            ),
+            (
+                r#"{"iss":"https://idp.example.org","sub":"5db2d1b6","aud":["a","b"],"scope":"openid"}"#,
+                None,
+                Some("https://idp.example.org"),
+            ),
+            (r#"{"sub":"5db2d1b6"}"#, None, None),
+            (r#"{"email":"[email protected]"}"#, Some("example.net"), None),
+        ] {
+            assert_eq!(
+                hints(&jwt(payload)),
+                Some((domain.map(str::to_string), issuer.map(str::to_string))),
+                "Unexpected claims for {payload}"
+            );
+        }
+    }
+
+    #[test]
+    fn non_jwt_tokens_are_ignored() {
+        for token in [
+            "sw1.eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJodHRwczovL2lkcC5leGFtcGxlLm9yZyJ9",
+            "sw1.eyJhbGciOiJSUzI1NiJ9",
+            "opaque-token",
+            "one.two",
+            "one.two.three.four",
+            "",
+        ] {
+            assert!(
+                JwtClaims::decode_payload(token).is_none(),
+                "Token {token:?} was parsed as a JWT"
+            );
+        }
+    }
+}

crates/common/src/auth/oauth/token.rs

@@ -17,7 +17,7 @@ pub const FAILED_TO_DECODE_TOKEN: &str = concat!(
     "the Authentication object."
 );
 
-const TOKEN_HEADER: &str = "sw1.";
+pub(crate) const TOKEN_HEADER: &str = "sw1.";
 const TOKEN_KEY_CONTEXT: &str = "stalwart-oauth-token-sw1";
 const OAUTH_EPOCH: u64 = 946684800; // Jan 1, 2000
 

crates/common/src/config/smtp/resolver.rs

@@ -214,6 +214,7 @@ impl Resolvers {
         let config_dnssec = resolver_config.clone();
         let mut opts_dnssec = opts.clone();
         opts_dnssec.validate = true;
+        opts_dnssec.num_concurrent_reqs = 1;
 
         let dnssec = DnssecResolver {
             resolver: TokioResolver::builder_with_config(
@@ -343,6 +344,7 @@ impl Default for Resolvers {
         let config_dnssec = config.clone();
         let mut opts_dnssec = opts.clone();
         opts_dnssec.validate = true;
+        opts_dnssec.num_concurrent_reqs = 1;
 
         Self {
             dns: MessageAuthenticator::new(config, opts).expect("Failed to build DNS resolver"),

crates/common/src/expr/functions/misc.rs

@@ -23,6 +23,13 @@ pub(crate) fn fn_is_number(v: Vec<Variable>) -> Variable {
     matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into()
 }
 
+pub(crate) fn fn_bit_and(v: Vec<Variable>) -> Variable {
+    match (v[0].to_integer(), v[1].to_integer()) {
+        (Some(lhs), Some(rhs)) => Variable::Integer(lhs & rhs),
+        _ => Variable::Integer(0),
+    }
+}
+
 pub(crate) fn fn_is_ip_addr(v: Vec<Variable>) -> Variable {
     v[0].to_string()
         .as_str()

crates/common/src/expr/functions/mod.rs

@@ -46,6 +46,7 @@ pub(crate) const FUNCTIONS: &[(&str, fn(Vec<Variable>) -> Variable, u32)] = &[
     ("email_part", email::fn_email_part, 2),
     ("is_empty", misc::fn_is_empty, 1),
     ("is_number", misc::fn_is_number, 1),
+    ("bit_and", misc::fn_bit_and, 2),
     ("is_ip_addr", misc::fn_is_ip_addr, 1),
     ("is_ipv4_addr", misc::fn_is_ipv4_addr, 1),
     ("is_ipv6_addr", misc::fn_is_ipv6_addr, 1),

crates/common/src/manager/application.rs

@@ -11,8 +11,11 @@ use registry::schema::{enums::CompressionAlgo, structs::Application};
 use std::{
     borrow::Cow,
     io::{self, Cursor, Read},
-    path::PathBuf,
-    sync::Arc,
+    path::{Path, PathBuf},
+    sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    },
     time::Duration,
 };
 use store::{
@@ -36,16 +39,18 @@ enum IndexEdit<'x> {
 pub struct WebApplications {
     applications: ArcSwap<Vec<WebApplicationManager>>,
     routes: ArcSwap<AHashMap<String, Arc<AppRoutes>>>,
+    generation: AtomicU64,
 }
 
 pub struct AppRoutes {
     resources: AHashMap<String, Resource<PathBuf>>,
     oauth_client_id_meta: Option<String>,
+    _bundle_dir: TempDir,
 }
 
 #[derive(Clone)]
 pub struct WebApplicationManager {
-    bundle_path: TempDir,
+    base_path: PathBuf,
     prefixes: Vec<String>,
     description: String,
     url: String,
@@ -79,6 +84,7 @@ impl WebApplications {
         Self {
             applications: ArcSwap::new(Arc::new(Vec::new())),
             routes: ArcSwap::new(Arc::new(AHashMap::new())),
+            generation: AtomicU64::new(0),
         }
     }
 
@@ -128,48 +134,55 @@ impl WebApplications {
     }
 
     pub async fn unpack_all(&self, server: &Server, update: bool) {
-        let mut routes = AHashMap::new();
+        let previous = self.routes.load_full();
+        let sweep_orphans = previous.is_empty();
+        let mut routes = AHashMap::with_capacity(previous.len());
+
         for app in self.applications.load().as_ref() {
-            if update && let Err(err) = app.delete(server).await {
-                trc::event!(
-                    Resource(trc::ResourceEvent::Error),
-                    Reason = err,
-                    Url = app.url.clone(),
-                    Details = format!(
-                        "Failed to delete application bundle for prefixes: {}",
-                        app.prefixes.join(", ")
-                    )
-                );
-            }
-            match app.unpack(server).await {
-                Ok(resources) => {
-                    let app_routes = Arc::new(AppRoutes {
-                        resources,
-                        oauth_client_id_meta: app
-                            .oauth_client_id
-                            .as_deref()
-                            .map(oauth_client_id_meta),
-                    });
+            match app
+                .unpack(server, self.next_generation(), update, sweep_orphans)
+                .await
+            {
+                Ok(app_routes) => {
+                    let app_routes = Arc::new(app_routes);
 
                     for prefix in &app.prefixes {
                         routes.insert(prefix.clone(), app_routes.clone());
                     }
                 }
                 Err(err) => {
+                    let mut is_retained = false;
+                    for prefix in &app.prefixes {
+                        if let Some(app_routes) = previous.get(prefix) {
+                            routes.insert(prefix.clone(), app_routes.clone());
+                            is_retained = true;
+                        }
+                    }
+
                     trc::event!(
                         Resource(trc::ResourceEvent::Error),
                         Reason = err,
                         Url = app.url.clone(),
                         Details = format!(
-                            "Failed to unpack application for prefixes: {}",
-                            app.prefixes.join(", ")
+                            "Failed to unpack application for prefixes: {}, {}",
+                            app.prefixes.join(", "),
+                            if is_retained {
+                                "the previously unpacked bundle remains in service"
+                            } else {
+                                "no bundle is available to serve"
+                            }
                         )
                     );
                 }
             }
         }
+
         self.routes.store(Arc::new(routes));
     }
+
+    fn next_generation(&self) -> u64 {
+        self.generation.fetch_add(1, Ordering::Relaxed)
+    }
 }
 
 impl WebApplicationManager {
@@ -182,7 +195,7 @@ impl WebApplicationManager {
             .join(app.id.id().to_string());
 
         Self {
-            bundle_path: TempDir::new(base_path),
+            base_path,
             blob_key: BlobHash::generate(format!("{}{}", APP_BLOB_PREFIX, app.id.id()).as_bytes()),
             url: app.object.resource_url,
             description: app.object.description,
@@ -202,82 +215,43 @@ impl WebApplicationManager {
         }
     }
 
-    async fn unpack(&self, server: &Server) -> trc::Result<AHashMap<String, Resource<PathBuf>>> {
-        // Delete any existing bundles
-        self.bundle_path.clean().await.map_err(unpack_error)?;
-
-        // Obtain application bundle
-        let bundle = if let Some(bundle) = server
-            .blob_store()
-            .get_blob(self.blob_key.as_slice(), 0..usize::MAX)
-            .await?
-        {
-            bundle
+    async fn unpack(
+        &self,
+        server: &Server,
+        generation: u64,
+        force_refresh: bool,
+        sweep_orphans: bool,
+    ) -> trc::Result<AppRoutes> {
+        let cached = if force_refresh {
+            None
         } else {
-            // Fetch app bundle
-            let resource = fetch_resource(&self.url, None, Duration::from_secs(60), MAX_APP_SIZE)
-                .await
-                .map_err(|err| {
-                    trc::ResourceEvent::Error
-                        .caused_by(trc::location!())
-                        .ctx(Key::Url, self.url.clone())
-                        .reason(err)
-                        .details("Failed to fetch application bundle")
-                })?;
-
-            // Store in blob store for future use
             server
                 .blob_store()
-                .put_blob(self.blob_key.as_slice(), &resource, CompressionAlgo::None)
-                .await
-                .caused_by(trc::location!())?;
-
-            // Schedule expiration
-            let mut batch = BatchBuilder::new();
-            batch
-                .set(
-                    BlobOp::Link {
-                        hash: self.blob_key.clone(),
-                        to: BlobLink::Temporary {
-                            until: now() + self.expiry,
-                        },
-                    },
-                    vec![],
-                )
-                .set(
-                    BlobOp::Commit {
-                        hash: self.blob_key.clone(),
-                    },
-                    Vec::new(),
-                );
-            server
-                .store()
-                .write(batch.build_all())
-                .await
-                .caused_by(trc::location!())?;
-
-            trc::event!(
-                Resource(trc::ResourceEvent::ApplicationUpdated),
-                Url = self.url.clone(),
-                Details = self.description.clone(),
-            );
-
-            resource
+                .get_blob(self.blob_key.as_slice(), 0..usize::MAX)
+                .await?
+        };
+        let is_cached = cached.is_some();
+        let bundle = match cached {
+            Some(bundle) => bundle,
+            None => self.fetch().await?,
         };
 
+        let staging = TempDir::new(self.base_path.join(format!("{:x}-{generation:x}", now())));
+        staging.create().await.map_err(unpack_error)?;
+
         let url = self.url.clone();
-        let bundle_path = self.bundle_path.path.clone();
-        let routes = tokio::task::spawn_blocking(move || -> trc::Result<_> {
-            let mut bundle = zip::ZipArchive::new(Cursor::new(bundle)).map_err(|err| {
+        let bundle_path = staging.path.clone();
+        let (resources, bundle) = tokio::task::spawn_blocking(move || -> trc::Result<_> {
+            let mut archive = zip::ZipArchive::new(Cursor::new(bundle)).map_err(|err| {
                 trc::ResourceEvent::Error
                     .caused_by(trc::location!())
                     .reason(err)
                     .ctx(Key::Url, url.clone())
                     .details("Failed to decompress application bundle")
             })?;
-            let mut routes = AHashMap::new();
-            for i in 0..bundle.len() {
-                let mut file = bundle.by_index(i).map_err(|err| {
+            let mut resources = AHashMap::with_capacity(archive.len());
+            for i in 0..archive.len() {
+                let mut file = archive.by_index(i).map_err(|err| {
                     trc::ResourceEvent::Error
                         .caused_by(trc::location!())
                         .reason(err)
@@ -315,9 +289,9 @@ impl WebApplicationManager {
                     contents: path,
                 };
 
-                routes.insert(file_name, resource);
+                resources.insert(file_name, resource);
             }
-            Ok(routes)
+            Ok((resources, archive.into_inner().into_inner()))
         })
         .await
         .map_err(|err| {
@@ -327,21 +301,81 @@ impl WebApplicationManager {
                 .details("Bundle unpack task panicked")
         })??;
 
+        if !is_cached && let Err(err) = self.cache(server, &bundle).await {
+            trc::event!(
+                Resource(trc::ResourceEvent::Error),
+                Reason = err,
+                Url = self.url.clone(),
+                Details = "Failed to cache application bundle, it will be downloaded again"
+            );
+        }
+
+        if sweep_orphans {
+            remove_siblings(&self.base_path, &staging.path).await;
+        }
+
         trc::event!(
             Resource(trc::ResourceEvent::ApplicationUnpacked),
             Url = self.url.clone(),
-            Path = self.bundle_path.path.to_string_lossy().into_owned(),
+            Path = staging.path.to_string_lossy().into_owned(),
         );
 
-        Ok(routes)
+        Ok(AppRoutes {
+            resources,
+            oauth_client_id_meta: self.oauth_client_id.as_deref().map(oauth_client_id_meta),
+            _bundle_dir: staging,
+        })
     }
 
-    async fn delete(&self, server: &Server) -> trc::Result<()> {
+    async fn fetch(&self) -> trc::Result<Vec<u8>> {
+        fetch_resource(&self.url, None, Duration::from_secs(60), MAX_APP_SIZE)
+            .await
+            .map_err(|err| {
+                trc::ResourceEvent::Error
+                    .caused_by(trc::location!())
+                    .ctx(Key::Url, self.url.clone())
+                    .reason(err)
+                    .details("Failed to fetch application bundle")
+            })
+    }
+
+    async fn cache(&self, server: &Server, bundle: &[u8]) -> trc::Result<()> {
         server
             .blob_store()
-            .delete_blob(self.blob_key.as_slice())
+            .put_blob(self.blob_key.as_slice(), bundle, CompressionAlgo::None)
             .await
-            .map(|_| ())
+            .caused_by(trc::location!())?;
+
+        let mut batch = BatchBuilder::new();
+        batch
+            .set(
+                BlobOp::Link {
+                    hash: self.blob_key.clone(),
+                    to: BlobLink::Temporary {
+                        until: now() + self.expiry,
+                    },
+                },
+                vec![],
+            )
+            .set(
+                BlobOp::Commit {
+                    hash: self.blob_key.clone(),
+                },
+                Vec::new(),
+            );
+        server
+            .store()
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+
+        trc::event!(
+            Resource(trc::ResourceEvent::ApplicationUpdated),
+            Url = self.url.clone(),
+            Details = self.description.clone(),
+        );
+
+        Ok(())
     }
 
     pub async fn delete_bundle(server: &Server, app_id: Id) -> trc::Result<()> {
@@ -361,7 +395,6 @@ impl Resource<Vec<u8>> {
     }
 }
 
-#[derive(Clone)]
 pub struct TempDir {
     pub path: PathBuf,
 }
@@ -371,11 +404,36 @@ impl TempDir {
         TempDir { path }
     }
 
-    pub async fn clean(&self) -> io::Result<()> {
+    pub async fn create(&self) -> io::Result<()> {
         if tokio::fs::metadata(&self.path).await.is_ok() {
             let _ = tokio::fs::remove_dir_all(&self.path).await;
         }
-        tokio::fs::create_dir(&self.path).await
+        tokio::fs::create_dir_all(&self.path).await
+    }
+}
+
+impl Drop for TempDir {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.path);
+    }
+}
+
+async fn remove_siblings(base_path: &Path, keep: &Path) {
+    let Ok(mut entries) = tokio::fs::read_dir(base_path).await else {
+        return;
+    };
+
+    while let Ok(Some(entry)) = entries.next_entry().await {
+        let path = entry.path();
+        if path == keep {
+            continue;
+        }
+
+        if matches!(entry.file_type().await, Ok(file_type) if file_type.is_dir()) {
+            let _ = tokio::fs::remove_dir_all(&path).await;
+        } else {
+            let _ = tokio::fs::remove_file(&path).await;
+        }
     }
 }
 
@@ -385,12 +443,6 @@ fn unpack_error(err: std::io::Error) -> trc::Error {
         .details("Failed to unpack application bundle")
 }
 
-impl Drop for TempDir {
-    fn drop(&mut self) {
-        let _ = std::fs::remove_dir_all(&self.path);
-    }
-}
-
 impl Default for WebApplications {
     fn default() -> Self {
         Self::new()
@@ -521,9 +573,9 @@ mod tests {
         );
     }
 
-    async fn fixture(name: &str, client_id: Option<&str>) -> (WebApplications, TempDir) {
+    async fn fixture(name: &str, client_id: Option<&str>) -> WebApplications {
         let dir = TempDir::new(std::env::temp_dir().join(format!("stalwart-app-{name}")));
-        dir.clean().await.unwrap();
+        dir.create().await.unwrap();
         tokio::fs::write(dir.path.join("index.html"), INDEX)
             .await
             .unwrap();
@@ -544,6 +596,7 @@ mod tests {
         let routes = Arc::new(AppRoutes {
             resources,
             oauth_client_id_meta: client_id.map(oauth_client_id_meta),
+            _bundle_dir: dir,
         });
 
         let mut map = AHashMap::new();
@@ -553,7 +606,7 @@ mod tests {
         let apps = WebApplications::new();
         apps.routes.store(Arc::new(map));
 
-        (apps, dir)
+        apps
     }
 
     async fn serve_html(apps: &WebApplications, prefix: &str, path: &str) -> String {
@@ -565,7 +618,7 @@ mod tests {
 
     #[tokio::test]
     async fn serving_index_injects_the_prefix_and_client_id() {
-        let (apps, _dir) = fixture("serve-configured", Some("pocket-id-client")).await;
+        let apps = fixture("serve-configured", Some("pocket-id-client")).await;
 
         let html = serve_html(&apps, "admin", "index.html").await;
         assert!(html.contains("<base href=\"/admin/\" />"), "{html}");
@@ -584,7 +637,7 @@ mod tests {
 
     #[tokio::test]
     async fn unknown_paths_fall_back_to_a_rewritten_index() {
-        let (apps, _dir) = fixture("serve-fallback", Some("pocket-id-client")).await;
+        let apps = fixture("serve-fallback", Some("pocket-id-client")).await;
 
         let html = serve_html(&apps, "admin", "settings/directory").await;
         assert!(html.contains("<base href=\"/admin/\" />"), "{html}");
@@ -596,7 +649,7 @@ mod tests {
 
     #[tokio::test]
     async fn assets_and_unknown_prefixes_are_untouched() {
-        let (apps, _dir) = fixture("serve-assets", Some("pocket-id-client")).await;
+        let apps = fixture("serve-assets", Some("pocket-id-client")).await;
 
         let served = apps.serve("admin", "app.js").await.unwrap().unwrap();
         assert_eq!(served.resource.contents, b"export const x = 1;\n");
@@ -608,7 +661,7 @@ mod tests {
 
     #[tokio::test]
     async fn serving_index_without_a_client_id_keeps_the_placeholder() {
-        let (apps, _dir) = fixture("serve-unconfigured", None).await;
+        let apps = fixture("serve-unconfigured", None).await;
 
         let html = serve_html(&apps, "admin", "index.html").await;
         assert!(html.contains("<base href=\"/admin/\" />"), "{html}");
@@ -624,4 +677,65 @@ mod tests {
 
         assert_eq!(rewrite_index(bundle, "admin", None), bundle.as_bytes());
     }
+    #[tokio::test]
+    async fn missing_parent_directories_are_created() {
+        let base = std::env::temp_dir().join("stalwart-app-nested");
+        let _ = tokio::fs::remove_dir_all(&base).await;
+
+        let dir = TempDir::new(base.join("webui").join("0"));
+        dir.create().await.unwrap();
+
+        assert!(tokio::fs::metadata(&dir.path).await.is_ok());
+
+        drop(dir);
+        let _ = tokio::fs::remove_dir_all(&base).await;
+    }
+
+    #[tokio::test]
+    async fn dropping_the_routes_removes_the_bundle_directory() {
+        let apps = fixture("drop-guard", None).await;
+        let path = apps
+            .routes
+            .load()
+            .get("admin")
+            .unwrap()
+            ._bundle_dir
+            .path
+            .clone();
+
+        assert!(tokio::fs::metadata(&path).await.is_ok());
+
+        apps.routes.store(Arc::new(AHashMap::new()));
+
+        assert!(tokio::fs::metadata(&path).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn sweeping_orphans_spares_the_current_generation() {
+        let base = std::env::temp_dir().join("stalwart-app-sweep");
+        let _ = tokio::fs::remove_dir_all(&base).await;
+
+        let current = TempDir::new(base.join("1"));
+        current.create().await.unwrap();
+        let orphan = base.join("0");
+        tokio::fs::create_dir_all(&orphan).await.unwrap();
+        let stray = base.join("webui.zip");
+        tokio::fs::write(&stray, b"not a bundle").await.unwrap();
+
+        remove_siblings(&base, &current.path).await;
+
+        assert!(tokio::fs::metadata(&current.path).await.is_ok());
+        assert!(tokio::fs::metadata(&orphan).await.is_err());
+        assert!(tokio::fs::metadata(&stray).await.is_err());
+
+        drop(current);
+        let _ = tokio::fs::remove_dir_all(&base).await;
+    }
+
+    #[test]
+    fn generations_never_repeat() {
+        let apps = WebApplications::new();
+
+        assert_ne!(apps.next_generation(), apps.next_generation());
+    }
 }

crates/common/src/network/acme/order.rs

@@ -96,7 +96,38 @@ impl AcmeRequestBuilder {
         reuse_key_pem: Option<String>,
         dns_parameters: Option<AcmeDnsParameters>,
     ) -> AcmeResult<PemCert> {
-        let mut params = CertificateParams::new(domains.clone()).map_err(|err| {
+        let mut published = BTreeSet::new();
+        let result = self
+            .run_order(
+                server,
+                &domains,
+                reuse_key_pem,
+                dns_parameters.as_ref(),
+                &mut published,
+            )
+            .await;
+
+        if let Some(dns_parameters) = &dns_parameters {
+            for (zone, challenge_name) in published {
+                let _ = dns_parameters
+                    .updater
+                    .delete_rrset(&zone, &challenge_name, dns_update::DnsRecordType::TXT)
+                    .await;
+            }
+        }
+
+        result
+    }
+
+    async fn run_order(
+        &self,
+        server: &Server,
+        domains: &[String],
+        reuse_key_pem: Option<String>,
+        dns_parameters: Option<&AcmeDnsParameters>,
+        published: &mut BTreeSet<(String, String)>,
+    ) -> AcmeResult<PemCert> {
+        let mut params = CertificateParams::new(domains.to_vec()).map_err(|err| {
             AcmeError::Crypto(format!("Failed to create certificate params: {}", err))
         })?;
         params.distinguished_name = DistinguishedName::new();
@@ -108,7 +139,7 @@ impl AcmeRequestBuilder {
                 AcmeError::Crypto(format!("Failed to generate key pair: {}", err))
             })?,
         };
-        let response = self.new_order(domains.clone()).await?;
+        let response = self.new_order(domains.to_vec()).await?;
         let order_url = response.location;
         let mut order = response.body;
         let mut retry_after = None;
@@ -117,7 +148,7 @@ impl AcmeRequestBuilder {
             Acme(AcmeEvent::OrderStart),
             Url = self.directory.new_order.to_string(),
             Details = order_url.to_string(),
-            Hostname = domains.as_slice(),
+            Hostname = domains,
             Type = self.challenge.as_str(),
         );
 
@@ -126,19 +157,20 @@ impl AcmeRequestBuilder {
                 OrderStatus::Pending => {
                     if matches!(self.challenge, ChallengeType::Dns01) {
                         for url in &order.authorizations {
-                            self.authorize(server, url, dns_parameters.as_ref()).await?;
+                            self.authorize(server, url, dns_parameters, Some(published))
+                                .await?;
                         }
                     } else {
                         let auth_futures = order
                             .authorizations
                             .iter()
-                            .map(|url| self.authorize(server, url, dns_parameters.as_ref()));
+                            .map(|url| self.authorize(server, url, dns_parameters, None));
                         try_join_all(auth_futures).await?;
                     }
                     trc::event!(
                         Acme(AcmeEvent::AuthCompleted),
                         Url = self.directory.new_order.to_string(),
-                        Hostname = domains.as_slice(),
+                        Hostname = domains,
                     );
                     let response = self.order(&order_url).await?;
                     order = response.body;
@@ -149,7 +181,7 @@ impl AcmeRequestBuilder {
                         trc::event!(
                             Acme(AcmeEvent::OrderProcessing),
                             Url = self.directory.new_order.to_string(),
-                            Hostname = domains.as_slice(),
+                            Hostname = domains,
                             Total = i,
                         );
 
@@ -177,7 +209,7 @@ impl AcmeRequestBuilder {
                     trc::event!(
                         Acme(AcmeEvent::OrderReady),
                         Url = self.directory.new_order.to_string(),
-                        Hostname = domains.as_slice(),
+                        Hostname = domains,
                     );
 
                     let csr = params.serialize_request(&key_pair).map_err(|err| {
@@ -190,10 +222,10 @@ impl AcmeRequestBuilder {
                     trc::event!(
                         Acme(AcmeEvent::OrderValid),
                         Url = self.directory.new_order.to_string(),
-                        Hostname = domains.as_slice(),
+                        Hostname = domains,
                     );
 
-                    let certificate = self.select_certificate(&domains, certificate).await?;
+                    let certificate = self.select_certificate(domains, certificate).await?;
 
                     return Ok(PemCert {
                         certificate,
@@ -211,7 +243,7 @@ impl AcmeRequestBuilder {
                         Acme(AcmeEvent::OrderInvalid),
                         Url = self.directory.new_order.to_string(),
                         Details = order_url.to_string(),
-                        Hostname = domains.as_slice(),
+                        Hostname = domains,
                         Reason = reason.clone(),
                     );
 
@@ -226,6 +258,7 @@ impl AcmeRequestBuilder {
         server: &Server,
         url: &String,
         dns_parameters: Option<&AcmeDnsParameters>,
+        published: Option<&mut BTreeSet<(String, String)>>,
     ) -> AcmeResult<()> {
         let response = self
             .auth(url)
@@ -287,7 +320,12 @@ impl AcmeRequestBuilder {
                             .await?;
                     }
                     ChallengeType::Dns01 => {
-                        let dns_parameters = dns_parameters.unwrap();
+                        let Some(dns_parameters) = dns_parameters else {
+                            return Err(AcmeError::Invalid(
+                                "DNS-01 challenge requested but a DNS provider was not configured"
+                                    .to_string(),
+                            ));
+                        };
                         let domain = domain.strip_prefix("*.").unwrap_or(&domain);
 
                         let zone = dns_parameters
@@ -308,6 +346,11 @@ impl AcmeRequestBuilder {
                             )
                             .await
                             .map_err(AcmeError::Dns)?;
+
+                        if let Some(published) = published {
+                            published.insert((zone.to_string(), challenge_name.clone()));
+                        }
+
                         dns_parameters
                             .updater
                             .wait_for_txt_propagation(&challenge_name, zone, &proof)

crates/common/src/network/dns/update.rs

@@ -1150,6 +1150,36 @@ impl DnsUpdater {
         Ok(())
     }
 
+    pub async fn delete_rrset(
+        &self,
+        origin: &str,
+        name: &str,
+        record_type: DnsRecordType,
+    ) -> Result<(), String> {
+        if let Err(err) = self
+            .updater
+            .set_rrset(
+                name,
+                record_type,
+                self.ttl.as_secs() as u32,
+                Vec::new(),
+                origin,
+            )
+            .await
+        {
+            trc::event!(
+                Dns(DnsEvent::RecordDeletionFailed),
+                Hostname = name.to_string(),
+                Details = origin.to_string(),
+                Type = record_type.as_str(),
+                Reason = err.to_string(),
+            );
+            return Err(format!("Failed to delete DNS RRSet: {}", err));
+        }
+
+        Ok(())
+    }
+
     pub async fn add_to_rrset(
         &self,
         origin: &str,

crates/common/src/network/mta.rs

@@ -21,6 +21,7 @@ use crate::{
     manager::SPAM_CLASSIFIER_KEY,
     network::RcptResolution,
 };
+use ahash::AHashSet;
 use directory::Recipient;
 use mail_auth::IpLookupStrategy;
 use registry::schema::{enums::ExpressionVariable, structs::MaskedEmail};
@@ -36,6 +37,7 @@ use store::{
 };
 use trc::{AddContext, SpamEvent};
 use types::id::Id;
+use utils::DomainPart;
 
 impl Server {
     pub async fn rcpt_resolve(
@@ -130,7 +132,10 @@ impl Server {
                 }
                 EmailCache::MailingList(id) => {
                     if let Some(list) = self.try_list(id).await? {
-                        return Ok(RcptResolution::Expand(list.recipients.clone()));
+                        return Ok(RcptResolution::Expand(
+                            self.expand_nested_lists(id, list.recipients.clone())
+                                .await?,
+                        ));
                     } else {
                         self.inner
                             .cache
@@ -162,6 +167,56 @@ impl Server {
         }
     }
 
+    async fn expand_nested_lists(
+        &self,
+        list_id: u32,
+        recipients: Arc<[Box<str>]>,
+    ) -> trc::Result<Arc<[Box<str>]>> {
+        let mut has_nested = false;
+        for member in recipients.iter() {
+            if let Some(EmailCache::MailingList(_)) = self.rcpt_id_from_email(member).await? {
+                has_nested = true;
+                break;
+            }
+        }
+        if !has_nested {
+            return Ok(recipients);
+        }
+
+        let mut expanded = Vec::with_capacity(recipients.len());
+        let mut seen: AHashSet<Box<str>> = AHashSet::with_capacity(recipients.len());
+        let mut visited = AHashSet::from_iter([list_id]);
+        let mut pending: Vec<Arc<[Box<str>]>> = Vec::new();
+        let mut members = recipients;
+
+        loop {
+            for member in members.iter() {
+                if let Some(EmailCache::MailingList(nested_id)) =
+                    self.rcpt_id_from_email(member).await?
+                {
+                    if !visited.insert(nested_id) {
+                        continue;
+                    }
+                    if let Some(nested) = self.try_list(nested_id).await? {
+                        pending.push(nested.recipients.clone());
+                        continue;
+                    }
+                }
+
+                if seen.insert(member.to_canonical_address().into()) {
+                    expanded.push(member.clone());
+                }
+            }
+
+            let Some(next) = pending.pop() else {
+                break;
+            };
+            members = next;
+        }
+
+        Ok(expanded.into())
+    }
+
     pub async fn get_dkim_signers(
         &self,
         domain: &str,

crates/coordinator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "coordinator"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/dav-proto/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dav-proto"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/dav/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dav"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/directory/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "directory"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/email/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "email"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/email/src/message/delivery.rs

@@ -17,6 +17,8 @@ use std::{borrow::Cow, future::Future};
 use store::ahash::AHashMap;
 use types::blob_hash::BlobHash;
 
+pub const ORCPT_ADDR_TYPE: &str = "rfc822;";
+
 #[derive(Debug)]
 pub struct IngestMessage {
     pub sender_address: String,
@@ -35,6 +37,12 @@ pub struct IngestRecipient {
 }
 
 impl IngestRecipient {
+    pub fn orcpt_parameter(&self) -> Option<String> {
+        self.orcpt
+            .as_deref()
+            .map(|orcpt| format!("{ORCPT_ADDR_TYPE}{orcpt}"))
+    }
+
     pub fn is_spam(&self) -> bool {
         self.spam_percentage
             .is_some_and(|percentage| percentage >= 50)

crates/email/src/sieve/ingest.rs

@@ -124,6 +124,7 @@ impl SieveScriptIngest for Server {
             .caused_by(trc::location!())?;
 
         // Create Sieve instance
+        let orcpt = envelope_to.orcpt_parameter();
         let mut instance = self.core.sieve.untrusted_runtime.filter_parsed(message);
 
         // Set account name and email
@@ -139,7 +140,7 @@ impl SieveScriptIngest for Server {
         // Set envelope
         instance.set_envelope(Envelope::From, envelope_from);
         instance.set_envelope(Envelope::To, envelope_to.address.as_str());
-        if let Some(orcpt) = &envelope_to.orcpt {
+        if let Some(orcpt) = &orcpt {
             instance.set_envelope(Envelope::Orcpt, orcpt.as_str());
         }
         instance.set_spam_status(spam_status(envelope_to.spam_percentage));

crates/groupware/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "groupware"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/http-proto/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "http_proto"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/http/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "http"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/http/src/api/diagnose.rs

@@ -230,14 +230,7 @@ async fn delivery_diagnose(
 
     // Lookup MX
     let now = Instant::now();
-    let mxs = match server
-        .core
-        .smtp
-        .resolvers
-        .dns
-        .mx_lookup(&domain, Some(&server.inner.cache.dns_mx))
-        .await
-    {
+    let mxs = match server.mx_lookup(domain.as_str()).await {
         Ok(mxs) => mxs,
         Err(err) => {
             tx.send(DeliveryStage::MxLookupError {
@@ -419,7 +412,7 @@ async fn delivery_diagnose(
                 })
                 .await?;
 
-                None
+                continue 'outer;
             }
             Ok(TlsaResult::Missing) => {
                 tx.send(DeliveryStage::TlsaNotFound {
@@ -440,14 +433,17 @@ async fn delivery_diagnose(
                         reason: "No TLSA records found for MX".to_string(),
                     })
                     .await?;
+
+                    None
                 } else {
                     tx.send(DeliveryStage::TlsaLookupError {
                         elapsed: now.elapsed_ms(),
                         reason: err.to_string(),
                     })
                     .await?;
+
+                    continue 'outer;
                 }
-                None
             }
         };
 

crates/imap-proto/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imap_proto"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/imap/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imap"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/imap/src/op/copy_move.rs

@@ -390,6 +390,16 @@ impl<T: SessionStream> SessionData<T> {
                 .await
                 .imap_ctx(&arguments.tag, trc::location!())?;
             let mut dest_cache = None;
+            let train_spam = if dest_mailbox_id == JUNK_ID {
+                Some(true)
+            } else if src_mailbox.id.mailbox_id == JUNK_ID && dest_mailbox_id != TRASH_ID {
+                Some(false)
+            } else {
+                None
+            };
+            let mut train_batch = BatchBuilder::new();
+            let mut did_train = false;
+            train_batch.with_account_id(src_account_id);
             for (id, imap_id) in ids {
                 match self
                     .server
@@ -515,11 +525,33 @@ impl<T: SessionStream> SessionData<T> {
                     }
                 };
 
+                if let Some(is_spam) = train_spam {
+                    self.server
+                        .add_account_spam_sample(
+                            &mut train_batch,
+                            src_account_id,
+                            id,
+                            is_spam,
+                            self.session_id,
+                        )
+                        .await
+                        .imap_ctx(&arguments.tag, trc::location!())?;
+                    train_batch.commit_point();
+                    did_train = true;
+                }
+
                 if is_move {
                     destroy_ids.insert(id);
                 }
             }
 
+            if did_train {
+                self.server
+                    .commit_batch(train_batch)
+                    .await
+                    .imap_ctx(&arguments.tag, trc::location!())?;
+            }
+
             // Untag or delete emails
             if !destroy_ids.is_empty() {
                 let mut batch = BatchBuilder::new();

crates/jmap-proto/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jmap_proto"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/jmap/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jmap"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/jmap/src/email/copy.rs

@@ -11,7 +11,11 @@ use crate::{
 use common::{Server, auth::AccessToken};
 use email::{
     cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess},
-    message::copy::{CopyMessageError, EmailCopy},
+    mailbox::JUNK_ID,
+    message::{
+        copy::{CopyMessageError, EmailCopy},
+        ingest::EmailIngest,
+    },
 };
 use http_proto::HttpSessionData;
 use jmap_proto::{
@@ -29,6 +33,7 @@ use jmap_proto::{
 };
 use jmap_tools::{Key, Value};
 use std::future::Future;
+use store::write::BatchBuilder;
 use trc::AddContext;
 use types::acl::Acl;
 use utils::map::vec_map::VecMap;
@@ -87,6 +92,9 @@ impl JmapEmailCopy for Server {
         };
         let on_success_delete = request.on_success_destroy_original.unwrap_or(false);
         let mut destroy_ids = Vec::new();
+        let mut train_batch = BatchBuilder::new();
+        let mut did_train = false;
+        train_batch.with_account_id(from_account_id);
 
         'create: for (id, create) in request.create.into_valid() {
             let mut from_message_id = None;
@@ -208,6 +216,7 @@ impl JmapEmailCopy for Server {
             }
 
             // Add response
+            let train_spam = mailboxes.contains(&JUNK_ID);
             match self
                 .copy_message(
                     from_account_id,
@@ -221,6 +230,20 @@ impl JmapEmailCopy for Server {
                 .await?
             {
                 Ok(email) => {
+                    if train_spam {
+                        self.add_account_spam_sample(
+                            &mut train_batch,
+                            from_account_id,
+                            from_message_id.document_id(),
+                            true,
+                            session.session_id,
+                        )
+                        .await
+                        .caused_by(trc::location!())?;
+                        train_batch.commit_point();
+                        did_train = true;
+                    }
+
                     response
                         .created
                         .append(id, ingested_into_object(email).into());
@@ -245,6 +268,12 @@ impl JmapEmailCopy for Server {
             }
         }
 
+        if did_train {
+            self.commit_batch(train_batch)
+                .await
+                .caused_by(trc::location!())?;
+        }
+
         // Update state
         if !response.created.is_empty() {
             response.new_state = self.get_cached_messages(account_id).await?.get_state(false);

crates/main/Cargo.toml

@@ -7,7 +7,7 @@ homepage = "https://stalw.art"
 keywords = ["imap", "jmap", "smtp", "email", "mail", "webdav", "server"]
 categories = ["email"]
 license = "AGPL-3.0-only OR LicenseRef-SEL"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [[bin]]

crates/main/src/test_data.rs

@@ -57,7 +57,7 @@ pub async fn insert_test_data(server: &Server) {
             server.inner.data.queue_id_gen.generate(),
             QueueName::default(),
         );
-        assert!(qm.save_changes(server, None).await);
+        assert!(qm.save_changes(server, None, None).await);
     }
 
     for report in sample_tls_internal_reports() {
@@ -163,7 +163,7 @@ fn sample_queued_messages(blob_hashes: Vec<BlobHash>) -> Vec<Message> {
                         },
                     }),
                     flags: RCPT_DSN_SENT,
-                    orcpt: Some("rfc822;[email protected]".into()),
+                    orcpt: Some("[email protected]".into()),
                 },
             ],
             received_from_ip: std::net::IpAddr::V4(Ipv4Addr::new(192, 168, 1, 10)),

crates/managesieve/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "managesieve"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/migration/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "migration"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/nlp/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nlp"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/pop3/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pop3"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/pop3/src/op/fetch.rs

@@ -64,14 +64,8 @@ impl<T: SessionStream> Session<T> {
                         )
                         .get_full_range();
 
-                    self.write_bytes(
-                        Response::Message::<u32> {
-                            bytes,
-                            lines: lines.unwrap_or(0),
-                        }
-                        .serialize(),
-                    )
-                    .await
+                    self.write_bytes(Response::Message::<u32> { bytes, lines }.serialize())
+                        .await
                 } else {
                     Err(trc::Pop3Event::Error
                         .into_err()

crates/pop3/src/protocol/response.rs

@@ -14,7 +14,7 @@ pub enum Response<'x, T> {
     List(Vec<T>),
     Message {
         bytes: SliceRange<'x>,
-        lines: u32,
+        lines: Option<u32>,
     },
     Capability {
         mechanisms: Vec<Mechanism>,
@@ -52,40 +52,65 @@ impl<'x, T: Display> Response<'x, T> {
                 buf
             }
             Response::Message { bytes, lines } => {
-                let mut buf = Vec::with_capacity(bytes.len() + 10);
-                buf.extend_from_slice(b"+OK ");
-                buf.extend_from_slice(bytes.len().to_string().as_bytes());
-                buf.extend_from_slice(b" octets\r\n");
-
-                let mut line_count = 0;
-                let mut last_byte = 0;
+                let lines = *lines;
+                let mut message = Vec::with_capacity(bytes.len() + 16);
+                let mut octets = 0;
+                let mut last_byte = b'\n';
+                let mut in_headers = lines.is_some();
+                let mut is_blank_line = true;
+                let mut body_lines = 0;
 
                 // Transparency procedure
                 for &byte in bytes.into_iter() {
                     // POP3 requires that lines end with CRLF, do this check to ensure that
                     if byte == b'\n' && last_byte != b'\r' {
-                        buf.push(b'\r');
+                        message.push(b'\r');
+                        octets += 1;
                     }
 
                     if byte == b'.' && last_byte == b'\n' {
-                        buf.push(b'.');
+                        message.push(b'.');
                     }
-                    buf.push(byte);
+                    message.push(byte);
+                    octets += 1;
                     last_byte = byte;
 
-                    if *lines > 0 && byte == b'\n' {
-                        line_count += 1;
-                        if line_count == *lines {
-                            break;
+                    match byte {
+                        b'\n' => {
+                            if in_headers {
+                                in_headers = !is_blank_line;
+                            } else {
+                                body_lines += 1;
+                            }
+                            if !in_headers && lines.is_some_and(|lines| body_lines >= lines) {
+                                break;
+                            }
+                            is_blank_line = true;
+                        }
+                        b'\r' => {}
+                        _ => {
+                            is_blank_line = false;
                         }
                     }
                 }
 
                 if last_byte != b'\n' {
-                    buf.extend_from_slice(b"\r\n");
+                    message.extend_from_slice(b"\r\n");
+                    octets += 2;
                 }
 
-                buf.extend_from_slice(b".\r\n");
+                if in_headers {
+                    message.extend_from_slice(b"\r\n");
+                    octets += 2;
+                }
+
+                message.extend_from_slice(b".\r\n");
+
+                let mut buf = Vec::with_capacity(message.len() + 24);
+                buf.extend_from_slice(b"+OK ");
+                buf.extend_from_slice(octets.to_string().as_bytes());
+                buf.extend_from_slice(b" octets\r\n");
+                buf.extend_from_slice(&message);
                 buf
             }
             Response::Capability { mechanisms, stls } => {
@@ -206,9 +231,51 @@ mod tests {
             (
                 Response::Message {
                     bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"),
-                    lines: 0,
+                    lines: None,
                 },
-                "+OK 35 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n",
+                "+OK 37 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n",
+            ),
+            (
+                Response::Message {
+                    bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"),
+                    lines: Some(0),
+                },
+                "+OK 17 octets\r\nSubject: test\r\n\r\n.\r\n",
+            ),
+            (
+                Response::Message {
+                    bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"),
+                    lines: Some(2),
+                },
+                "+OK 27 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n.\r\n",
+            ),
+            (
+                Response::Message {
+                    bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"),
+                    lines: Some(100),
+                },
+                "+OK 37 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n",
+            ),
+            (
+                Response::Message {
+                    bytes: SliceRange::Single(b"Subject: test\n\nbody\n"),
+                    lines: None,
+                },
+                "+OK 23 octets\r\nSubject: test\r\n\r\nbody\r\n.\r\n",
+            ),
+            (
+                Response::Message {
+                    bytes: SliceRange::Single(b"Subject: test\n\n.leading dot\n"),
+                    lines: Some(1),
+                },
+                "+OK 31 octets\r\nSubject: test\r\n\r\n..leading dot\r\n.\r\n",
+            ),
+            (
+                Response::Message {
+                    bytes: SliceRange::Single(b".dot\r\nSubject: test\r\n"),
+                    lines: Some(3),
+                },
+                "+OK 23 octets\r\n..dot\r\nSubject: test\r\n\r\n.\r\n",
             ),
         ] {
             assert_eq!(expected, String::from_utf8(cmd.serialize()).unwrap());

crates/registry/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "registry"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/scim-proto/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scim-proto"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/scim/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scim"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/services/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "services"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/services/src/broadcast/subscriber.rs

@@ -94,6 +94,13 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
                                     }
                                 };
 
+                                inner
+                                    .shared_core
+                                    .load()
+                                    .storage
+                                    .data
+                                    .invalidate_read_snapshot();
+
                                 loop {
                                     match batch.next_event() {
                                         Ok(Some(event)) => {
@@ -155,9 +162,7 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
                                                             .await;
                                                 }
                                                 BroadcastEvent::QueueRefresh => {
-                                                    let core = inner.shared_core.load_full();
-                                                    if core.network.roles.outbound_mta {
-                                                        core.storage.data.invalidate_read_snapshot();
+                                                    if inner.shared_core.load().network.roles.outbound_mta {
                                                         let _ = inner
                                                                 .ipc
                                                                 .queue_tx
@@ -166,9 +171,7 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
                                                     }
                                                 }
                                                 BroadcastEvent::RegistryChange(change) => {
-                                                    let server = inner.build_server();
-                                                    server.store().invalidate_read_snapshot();
-                                                    match Box::pin(server.reload_registry(change)).await {
+                                                    match Box::pin(inner.build_server().reload_registry(change)).await {
                                                         Ok(result) => {
                                                             result.log();
                                                         }

crates/services/src/task_manager/destroy_account.rs

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use crate::task_manager::TaskResult;
+use crate::task_manager::{TaskResult, deferred_retry_time};
 use common::Server;
 use email::{message::metadata::MessageMetadata, sieve::SieveScript};
 use groupware::file::FileNode;
@@ -39,7 +39,7 @@ impl DestroyAccountTask for Server {
         match destroy_account(self, task).await {
             Ok(result) => result,
             Err(err) => {
-                let result = TaskResult::temporary(err.to_string());
+                let result = TaskResult::deferred(deferred_retry_time(&err), err.to_string());
                 trc::error!(
                     err.account_id(task.account_id.document_id())
                         .details("Failed to destroy account")

crates/services/src/task_manager/index.rs

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use crate::task_manager::{Task, TaskDetails, TaskFailureType, TaskResult};
+use crate::task_manager::{Task, TaskDetails, TaskFailureType, TaskResult, deferred_retry_time};
 use common::Server;
 use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata};
 use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard};
@@ -255,7 +255,7 @@ impl SearchIndexTask for Server {
             );
             for r in results.iter_mut() {
                 if r.task_type == TaskType::Insert && r.result.is_success() {
-                    r.result = search_store_failure(retry_at, "Failed to index documents");
+                    r.result = TaskResult::deferred(retry_at, "Failed to index documents");
                 }
             }
             return results;
@@ -312,7 +312,7 @@ impl SearchIndexTask for Server {
                 for r in results.iter_mut() {
                     if r.task_type == TaskType::Delete && r.result.is_success() {
                         r.result =
-                            search_store_failure(retry_at, "Failed to delete documents from index");
+                            TaskResult::deferred(retry_at, "Failed to delete documents from index");
                     }
                 }
                 return results;
@@ -426,22 +426,6 @@ pub(crate) async fn reindex_account(server: &Server, account_id: u32) -> trc::Re
     Ok(())
 }
 
-fn deferred_retry_time(err: &trc::Error) -> Option<u64> {
-    err.value(trc::Key::NextRetry)
-        .and_then(|value| value.to_uint())
-}
-
-fn search_store_failure(retry_at: Option<u64>, message: &'static str) -> TaskResult {
-    match retry_at {
-        Some(retry_at) => TaskResult::Failure {
-            typ: TaskFailureType::Retry(retry_at),
-            message: message.into(),
-            max_attempts: None,
-        },
-        None => TaskResult::temporary(message),
-    }
-}
-
 fn attempt_number(status: &TaskStatus) -> u64 {
     match status {
         TaskStatus::Pending(_) => 0,

crates/services/src/task_manager/manager.rs

@@ -621,6 +621,7 @@ pub fn perpetual_retry_time(typ: TaskType, attempt: u64) -> Option<u64> {
             | TaskType::DkimManagement
             | TaskType::IndexDocument
             | TaskType::UnindexDocument
+            | TaskType::DestroyAccount
     )
     .then(|| {
         now().saturating_add(

crates/services/src/task_manager/mod.rs

@@ -134,4 +134,20 @@ impl TaskResult {
             max_attempts: None,
         }
     }
+
+    pub fn deferred(retry_at: Option<u64>, message: impl Into<String>) -> Self {
+        match retry_at {
+            Some(retry_at) => TaskResult::Failure {
+                typ: TaskFailureType::Retry(retry_at),
+                message: message.into(),
+                max_attempts: None,
+            },
+            None => TaskResult::temporary(message),
+        }
+    }
+}
+
+pub(crate) fn deferred_retry_time(err: &trc::Error) -> Option<u64> {
+    err.value(trc::Key::NextRetry)
+        .and_then(|value| value.to_uint())
 }

crates/smtp/Cargo.toml

@@ -7,7 +7,7 @@ homepage = "https://stalw.art/smtp"
 keywords = ["smtp", "email", "mail", "server"]
 categories = ["email"]
 license = "AGPL-3.0-only OR LicenseRef-SEL"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/smtp/src/core/mod.rs

@@ -11,6 +11,7 @@ use common::{
     config::smtp::auth::VerifyStrategy,
     network::{ServerInstance, asn::AsnGeoLookupResult},
 };
+use email::message::delivery::ORCPT_ADDR_TYPE;
 use mail_auth::{IprevOutput, SpfOutput};
 use smtp_proto::request::receiver::{
     BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver,
@@ -306,10 +307,13 @@ impl SessionAddress {
         }
     }
 
-    pub fn report_address(&self) -> &str {
+    pub fn orig_address(&self) -> &str {
+        self.dsn_info.as_deref().unwrap_or(&self.address_lcase)
+    }
+
+    pub fn orcpt_parameter(&self) -> Option<String> {
         self.dsn_info
-            .as_ref()
-            .and_then(|v| v.strip_prefix("rfc822;"))
-            .unwrap_or(&self.address_lcase)
+            .as_deref()
+            .map(|orcpt| format!("{ORCPT_ADDR_TYPE}{}", orcpt.to_lowercase()))
     }
 }

crates/smtp/src/inbound/data.rs

@@ -441,7 +441,7 @@ impl<T: SessionStream> Session<T> {
             if !rc.analysis.forward {
                 self.data
                     .rcpt_to
-                    .retain(|rcpt| !rc.analysis.is_report_address(rcpt.report_address()));
+                    .retain(|rcpt| !rc.analysis.is_report_address(rcpt.orig_address()));
             }
 
             if self.data.rcpt_to.is_empty() {

crates/smtp/src/inbound/rcpt.rs

@@ -202,8 +202,8 @@ impl<T: SessionStream> Session<T> {
                 let mut new_addr = SessionAddress::new(address);
 
                 if !self.data.rcpt_to.contains(&new_addr) {
-                    new_addr.dsn_info = format!("rfc822;{}", orig_addr.address_lcase).into();
                     new_addr.flags = orig_addr.flags;
+                    new_addr.dsn_info = orig_addr.address_lcase.into();
                     self.data.rcpt_to.push(new_addr);
                 } else {
                     trc::event!(
@@ -353,7 +353,6 @@ impl<T: SessionStream> Session<T> {
         // Expand list
         if let Some(members) = rcpt_members {
             let list_addr = self.data.rcpt_to.pop().unwrap();
-            let orcpt = format!("rfc822;{}", list_addr.address_lcase);
             for member in members.as_ref() {
                 let member_lcase = member.to_lowercase();
                 let is_local = match self
@@ -399,7 +398,7 @@ impl<T: SessionStream> Session<T> {
                 if !self.data.rcpt_to.contains(&member_addr)
                     && member_addr.address_lcase != list_addr.address_lcase
                 {
-                    member_addr.dsn_info = orcpt.clone().into();
+                    member_addr.dsn_info = list_addr.address_lcase.clone().into();
                     member_addr.flags = list_addr.flags;
                     self.data.rcpt_to.push(member_addr);
                 }

crates/smtp/src/inbound/spam.rs

@@ -89,17 +89,7 @@ impl<T: SessionStream> Session<T> {
                 .iter()
                 .map(|r| r.address_lcase.as_str())
                 .collect(),
-            env_rcpt_orig_to: self
-                .data
-                .rcpt_to
-                .iter()
-                .map(|r| {
-                    r.dsn_info
-                        .as_deref()
-                        .and_then(|info| info.strip_prefix("rfc822;"))
-                        .unwrap_or(r.address_lcase.as_str())
-                })
-                .collect(),
+            env_rcpt_orig_to: self.data.rcpt_to.iter().map(|r| r.orig_address()).collect(),
             is_test: false,
             is_train: false,
         }

crates/smtp/src/outbound/delivery.rs

@@ -15,8 +15,8 @@ use crate::outbound::lookup::{DnsLookup, SourceIp};
 use crate::outbound::mta_sts::lookup::MtaStsLookup;
 use crate::outbound::mta_sts::verify::VerifyPolicy;
 use crate::outbound::{client::StartTlsResult, dane::verify::TlsaVerify};
-use crate::queue::dsn::SendDsn;
-use crate::queue::spool::SmtpSpool;
+use crate::queue::dsn::{DsnStatus, SendDsn};
+use crate::queue::spool::{DSN_RETRY, SmtpSpool};
 use crate::queue::throttle::IsAllowed;
 use crate::queue::{
     Error, FROM_REPORT, HostResponse, MessageWrapper, Metadata, QueueEnvelope, QueuedMessage,
@@ -155,7 +155,7 @@ impl QueuedMessage {
         let span_id = message.span_id;
 
         // Send any due Delivery Status Notifications
-        server.send_dsn(&mut message).await;
+        let dsn_status = server.send_dsn(&mut message).await;
 
         match has_pending_delivery {
             PendingDelivery::Yes(true)
@@ -163,21 +163,27 @@ impl QueuedMessage {
                     .message
                     .next_delivery_event(self.queue_name.into())
                     .is_some_and(|due| due <= now()) => {}
-            PendingDelivery::No => {
+            PendingDelivery::No if dsn_status == DsnStatus::Completed => {
                 trc::event!(
                     Delivery(DeliveryEvent::Completed),
                     SpanId = span_id,
                     Elapsed = trc::Value::Duration((now() - message.message.created) * 1000)
                 );
 
-                // All message recipients expired, do not re-queue. (DSN has been already sent)
+                // All message recipients expired, do not re-queue.
                 message.remove(&server, self.due.into()).await;
 
                 return QueueEventStatus::Completed;
             }
+            PendingDelivery::No => {
+                message
+                    .save_changes(&server, self.due.into(), Some(now() + DSN_RETRY))
+                    .await;
+                return QueueEventStatus::Deferred;
+            }
             _ => {
                 // Re-queue the message if its not yet due for delivery
-                message.save_changes(&server, self.due.into()).await;
+                message.save_changes(&server, self.due.into(), None).await;
                 return QueueEventStatus::Deferred;
             }
         }
@@ -208,7 +214,7 @@ impl QueuedMessage {
                     }
                 }
 
-                message.save_changes(&server, self.due.into()).await;
+                message.save_changes(&server, self.due.into(), None).await;
 
                 return QueueEventStatus::Deferred;
             }
@@ -1485,7 +1491,7 @@ impl QueuedMessage {
         }
 
         // Send Delivery Status Notifications
-        server.send_dsn(&mut message).await;
+        let dsn_status = server.send_dsn(&mut message).await;
 
         // Notify queue manager
         if message.message.next_event(None).is_some() {
@@ -1501,7 +1507,13 @@ impl QueuedMessage {
             );
 
             // Save changes to disk
-            message.save_changes(&server, self.due.into()).await;
+            message.save_changes(&server, self.due.into(), None).await;
+
+            QueueEventStatus::Deferred
+        } else if dsn_status == DsnStatus::Deferred {
+            message
+                .save_changes(&server, self.due.into(), Some(now() + DSN_RETRY))
+                .await;
 
             QueueEventStatus::Deferred
         } else {

crates/smtp/src/outbound/local.rs

@@ -120,7 +120,7 @@ impl MessageWrapper {
                     )
                     .await;
 
-                message
+                let _ = message
                     .queue(
                         QueueParams::new(&autogenerated.message, self.span_id, server)
                             .with_dkim_signers(dkim_signers)

crates/smtp/src/queue/dsn.rs

@@ -10,9 +10,10 @@ use super::{
     Recipient, Status,
 };
 use crate::inbound::dkim::DkimSign;
-use crate::queue::spool::QueueParams;
+use crate::queue::spool::{DSN_RETRY, QueueParams};
 use crate::queue::{MessageWrapper, UnexpectedResponse};
 use common::Server;
+use email::message::delivery::ORCPT_ADDR_TYPE;
 use mail_builder::MessageBuilder;
 use mail_builder::headers::HeaderType;
 use mail_builder::headers::content_type::ContentType;
@@ -25,16 +26,24 @@ use std::fmt::Write;
 use std::future::Future;
 use store::write::now;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DsnStatus {
+    Completed,
+    Deferred,
+}
+
 pub trait SendDsn: Sync + Send {
-    fn send_dsn(&self, message: &mut MessageWrapper) -> impl Future<Output = ()> + Send;
+    fn send_dsn(&self, message: &mut MessageWrapper) -> impl Future<Output = DsnStatus> + Send;
     fn log_dsn(&self, message: &MessageWrapper) -> impl Future<Output = ()> + Send;
 }
 
 impl SendDsn for Server {
-    async fn send_dsn(&self, message: &mut MessageWrapper) {
+    async fn send_dsn(&self, message: &mut MessageWrapper) -> DsnStatus {
         // Send DSN events
         self.log_dsn(message).await;
 
+        let mut status = DsnStatus::Completed;
+
         if !message.message.return_path.is_empty() {
             // Build DSN
             if let Some(dsn) = message.build_dsn(self).await {
@@ -51,12 +60,19 @@ impl SendDsn for Server {
                         message.span_id,
                     )
                     .await;
-                dsn_message
+                if dsn_message
                     .queue(
                         QueueParams::new(&dsn, message.span_id, self)
                             .with_dkim_signers(dkim_signers),
                     )
-                    .await;
+                    .await
+                {
+                    message.mark_dsn_sent();
+                } else {
+                    status = DsnStatus::Deferred;
+                }
+            } else {
+                message.mark_dsn_sent();
             }
         } else {
             // Handle double bounce
@@ -64,7 +80,9 @@ impl SendDsn for Server {
         }
 
         // Update next DSN notify times
-        message.update_next_dsn(self).await;
+        message.update_next_dsn(self, status).await;
+
+        status
     }
 
     async fn log_dsn(&self, message: &MessageWrapper) {
@@ -132,7 +150,7 @@ impl SendDsn for Server {
 const MAX_HEADER_SIZE: usize = 4096;
 
 impl MessageWrapper {
-    pub async fn build_dsn(&mut self, server: &Server) -> Option<Vec<u8>> {
+    pub async fn build_dsn(&self, server: &Server) -> Option<Vec<u8>> {
         let config = &server.core.smtp.queue;
         let now = now();
 
@@ -141,13 +159,12 @@ impl MessageWrapper {
         let mut txt_failed = String::new();
         let mut dsn = String::new();
 
-        for rcpt in &mut self.message.recipients {
+        for rcpt in &self.message.recipients {
             if rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) {
                 continue;
             }
             match &rcpt.status {
                 Status::Completed(response) => {
-                    rcpt.flags |= RCPT_DSN_SENT;
                     if !rcpt.has_flag(RCPT_NOTIFY_SUCCESS) {
                         continue;
                     }
@@ -164,7 +181,6 @@ impl MessageWrapper {
                     response.write_dsn_text(&rcpt.address, &mut txt_delay);
                 }
                 Status::PermanentFailure(response) => {
-                    rcpt.flags |= RCPT_DSN_SENT;
                     if !rcpt.has_flag(RCPT_NOTIFY_FAILURE) {
                         continue;
                     }
@@ -357,7 +373,7 @@ impl MessageWrapper {
             .into()
     }
 
-    pub async fn update_next_dsn(&mut self, server: &Server) {
+    pub async fn update_next_dsn(&mut self, server: &Server, status: DsnStatus) {
         let now = now();
         let mut notify_changes = Vec::new();
         for (rcpt_idx, rcpt) in self.message.recipients.iter().enumerate() {
@@ -366,6 +382,11 @@ impl MessageWrapper {
                 Status::TemporaryFailure(_) | Status::Scheduled
             ) && rcpt.notify.due <= now
             {
+                if status == DsnStatus::Deferred {
+                    notify_changes.push((rcpt_idx, 0, now + DSN_RETRY));
+                    continue;
+                }
+
                 let envelope = QueueEnvelope::new(&self.message, rcpt);
 
                 let queue_id = server
@@ -391,6 +412,19 @@ impl MessageWrapper {
         }
     }
 
+    fn mark_dsn_sent(&mut self) {
+        for rcpt in &mut self.message.recipients {
+            if !rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER)
+                && matches!(
+                    rcpt.status,
+                    Status::Completed(_) | Status::PermanentFailure(_)
+                )
+            {
+                rcpt.flags |= RCPT_DSN_SENT;
+            }
+        }
+    }
+
     fn handle_double_bounce(&mut self) {
         let mut is_double_bounce = Vec::with_capacity(0);
         let now = now();
@@ -523,7 +557,7 @@ impl Message {
 impl Recipient {
     fn write_dsn(&self, dsn: &mut String) {
         if let Some(orcpt) = &self.orcpt {
-            let _ = write!(dsn, "Original-Recipient: rfc822;{orcpt}\r\n");
+            let _ = write!(dsn, "Original-Recipient: {ORCPT_ADDR_TYPE}{orcpt}\r\n");
         }
         let _ = write!(dsn, "Final-Recipient: rfc822;{}\r\n", self.address);
     }

crates/smtp/src/queue/spool.rs

@@ -45,6 +45,7 @@ use utils::DomainPart;
 
 pub const LOCK_EXPIRY: u64 = 10 * 60; // 10 minutes
 pub const QUEUE_REFRESH: u64 = 5 * 60; // 5 minutes
+pub const DSN_RETRY: u64 = 5 * 60; // 5 minutes
 pub(crate) const INFINITE_LOCK: u64 = 60 * 60 * 24 * 365; // 1 year
 const CANDIDATE_OVERSCAN: usize = 4;
 const MAX_PREALLOCATED_CANDIDATES: usize = 1024;
@@ -370,6 +371,7 @@ pub(crate) struct QueueParams<'x, 'y> {
 }
 
 impl MessageWrapper {
+    #[must_use]
     pub(crate) async fn queue<'x, 'y>(mut self, mut params: QueueParams<'x, 'y>) -> bool {
         // Add DKIM signatures
         let dkim_headers = if params.dkim_signers.is_some() {
@@ -669,7 +671,12 @@ impl MessageWrapper {
         recipient.queue = queue.virtual_queue;
     }
 
-    pub async fn save_changes(mut self, server: &Server, prev_event: Option<u64>) -> bool {
+    pub async fn save_changes(
+        mut self,
+        server: &Server,
+        prev_event: Option<u64>,
+        retry_at: Option<u64>,
+    ) -> bool {
         // Release quota for completed deliveries
         let mut batch = BatchBuilder::new();
         self.release_quota(&mut batch);
@@ -684,7 +691,12 @@ impl MessageWrapper {
                 },
             )));
         }
-        for (queue_name, due) in self.message.next_events() {
+        let mut next_events = self.message.next_events();
+        if let Some(retry_at) = retry_at {
+            let due = next_events.entry(self.queue_name).or_insert(retry_at);
+            *due = std::cmp::min(*due, retry_at);
+        }
+        for (queue_name, due) in next_events {
             batch.set(
                 ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent {
                     due,

crates/smtp/src/reporting/dmarc.rs

@@ -316,9 +316,6 @@ impl<T: SessionStream> Session<T> {
         if let Some(dkim2_output) = dkim2_output {
             report_record = report_record.with_dkim2_output(dkim2_output);
         }
-        if let Some(spf_ehlo) = &self.data.spf_ehlo {
-            report_record = report_record.with_spf_output(spf_ehlo, SPFDomainScope::Helo);
-        }
         if let Some(spf_mail_from) = &self.data.spf_mail_from {
             report_record = report_record.with_spf_output(spf_mail_from, SPFDomainScope::MailFrom);
         }

crates/smtp/src/reporting/inbound.rs

@@ -41,7 +41,7 @@ impl<T: AsyncWrite + AsyncRead + Unpin> Session<T> {
         self.data
             .rcpt_to
             .iter()
-            .any(|addr| analysis.is_report_address(addr.report_address()))
+            .any(|addr| analysis.is_report_address(addr.orig_address()))
     }
 }
 

crates/smtp/src/reporting/send.rs

@@ -98,7 +98,7 @@ impl MtaReportSend for Server {
         let dkim_signers = self
             .eval_signers(sign_config, &message.message, parent_session_id)
             .await;
-        message
+        let _ = message
             .queue(
                 QueueParams::new(&report, parent_session_id, self).with_dkim_signers(dkim_signers),
             )
@@ -130,7 +130,7 @@ impl MtaReportSend for Server {
         } else {
             None
         };
-        message
+        let _ = message
             .queue(
                 QueueParams::new(&raw_message, parent_session_id, self)
                     .with_dkim_signers(dkim_signers),

crates/smtp/src/scripts/envelope.rs

@@ -12,6 +12,7 @@ use smtp_proto::{
 use utils::DomainPart;
 
 use crate::core::{SessionAddress, SessionData};
+use email::message::delivery::ORCPT_ADDR_TYPE;
 
 impl SessionData {
     pub fn apply_envelope_modification(&mut self, envelope: Envelope, value: String) {
@@ -111,7 +112,11 @@ impl SessionData {
             }
             Envelope::Orcpt => {
                 if let Some(rcpt_to) = self.rcpt_to.last_mut() {
-                    rcpt_to.dsn_info = value.into();
+                    rcpt_to.dsn_info = value
+                        .strip_prefix(ORCPT_ADDR_TYPE)
+                        .map(str::to_string)
+                        .unwrap_or(value)
+                        .into();
                 }
             }
             Envelope::Envid => {

crates/smtp/src/scripts/event_loop.rs

@@ -297,7 +297,7 @@ impl RunScript for Server {
                                     None
                                 };
 
-                                message
+                                let _ = message
                                     .queue(
                                         QueueParams::new(raw_message, session_id, self)
                                             .with_dkim_signers(dkim_signers)

crates/smtp/src/scripts/exec.rs

@@ -95,10 +95,8 @@ impl<T: SessionStream> Session<T> {
                     params
                         .envelope
                         .push((Envelope::To, rcpt.address_lcase.to_string().into()));
-                    if let Some(orcpt) = &rcpt.dsn_info {
-                        params
-                            .envelope
-                            .push((Envelope::Orcpt, orcpt.as_str().to_lowercase().into()));
+                    if let Some(orcpt) = rcpt.orcpt_parameter() {
+                        params.envelope.push((Envelope::Orcpt, orcpt.into()));
                     }
                 }
             } else {
@@ -109,10 +107,10 @@ impl<T: SessionStream> Session<T> {
 
                 for rcpt in &self.data.rcpt_to {
                     recipients.push(Variable::from(rcpt.address_lcase.to_string()));
-                    orcpts.push(match &rcpt.dsn_info {
+                    orcpts.push(match rcpt.orcpt_parameter() {
                         Some(orcpt) => {
                             has_orcpts = true;
-                            Variable::from(orcpt.as_str().to_lowercase())
+                            Variable::from(orcpt)
                         }
                         None => Variable::default(),
                     });

crates/spam-filter/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spam-filter"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/store/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "store"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/store/src/backend/foundationdb/mod.rs

@@ -71,7 +71,7 @@ impl ReadVersion {
     }
 
     fn expire(&self) {
-        self.obtained.store(0, Ordering::Release);
+        self.version.store(0, Ordering::Release);
     }
 
     fn try_begin_refresh(&self) -> Option<RefreshGuard<'_>> {

crates/store/src/backend/meili/main.rs

@@ -11,14 +11,12 @@ use crate::{
         CalendarSearchField, ContactSearchField, EmailSearchField, SearchField, SearchableField,
         TracingSearchField,
     },
-    write::now,
 };
 use registry::schema::structs;
 use reqwest::{Error, Response, Url};
 use serde_json::{Value, json};
 use std::{sync::Arc, time::Duration};
 
-const UNCONFIRMED_TASK_RECHECK_DELAY: u64 = 600;
 pub(crate) const MAX_TOTAL_HITS: u64 = 100_000;
 
 impl MeiliSearchStore {
@@ -300,18 +298,13 @@ impl MeiliSearchStore {
             }
         }
 
-        let err = trc::StoreEvent::MeilisearchError
-            .reason("Timed out waiting for Meilisearch task")
-            .id(task_uid);
-
-        Err(if self.task_fail_on_timeout {
-            err
+        if self.task_fail_on_timeout {
+            Err(trc::StoreEvent::MeilisearchError
+                .reason("Timed out waiting for Meilisearch task")
+                .id(task_uid))
         } else {
-            err.ctx(
-                trc::Key::NextRetry,
-                now().saturating_add(UNCONFIRMED_TASK_RECHECK_DELAY),
-            )
-        })
+            Ok(true)
+        }
     }
 }
 

crates/store/src/backend/redis/lookup.rs

@@ -6,36 +6,37 @@
 
 use super::{RedisPool, RedisStore, into_error};
 use crate::{Deserialize, write::now};
-use redis::AsyncCommands;
+use deadpool::managed::{Manager, Object, Pool};
+use redis::{AsyncCommands, RedisError, RedisResult, RetryMethod, Script};
+use std::sync::LazyLock;
+
+static INCR_EXPIRE: LazyLock<Script> = LazyLock::new(|| {
+    Script::new(
+        "redis.call('INCRBY', KEYS[1], ARGV[1])
+         redis.call('EXPIRE', KEYS[1], ARGV[2])
+         return redis.call('GET', KEYS[1])",
+    )
+});
 
 impl RedisStore {
     pub async fn key_set(&self, key: &[u8], value: &[u8], expires: Option<u64>) -> trc::Result<()> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.key_set_(
-                    pool.get().await.map_err(into_error)?.as_mut(),
-                    key,
-                    value,
-                    expires,
-                )
+                with_conn(pool, async |conn| {
+                    Self::key_set_(conn, key, value, expires).await
+                })
                 .await
             }
             RedisPool::Cluster(pool) => {
-                self.key_set_(
-                    pool.get().await.map_err(into_error)?.as_mut(),
-                    key,
-                    value,
-                    expires,
-                )
+                with_conn(pool, async |conn| {
+                    Self::key_set_(conn, key, value, expires).await
+                })
                 .await
             }
             RedisPool::Sentinel(pool) => {
-                self.key_set_(
-                    pool.get().await.map_err(into_error)?.as_mut(),
-                    key,
-                    value,
-                    expires,
-                )
+                with_conn(pool, async |conn| {
+                    Self::key_set_(conn, key, value, expires).await
+                })
                 .await
             }
         }
@@ -44,30 +45,21 @@ impl RedisStore {
     pub async fn key_incr(&self, key: &[u8], value: i64, expires: Option<u64>) -> trc::Result<i64> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.key_incr_(
-                    pool.get().await.map_err(into_error)?.as_mut(),
-                    key,
-                    value,
-                    expires,
-                )
+                with_conn(pool, async |conn| {
+                    Self::key_incr_(conn, key, value, expires).await
+                })
                 .await
             }
             RedisPool::Cluster(pool) => {
-                self.key_incr_(
-                    pool.get().await.map_err(into_error)?.as_mut(),
-                    key,
-                    value,
-                    expires,
-                )
+                with_conn(pool, async |conn| {
+                    Self::key_incr_(conn, key, value, expires).await
+                })
                 .await
             }
             RedisPool::Sentinel(pool) => {
-                self.key_incr_(
-                    pool.get().await.map_err(into_error)?.as_mut(),
-                    key,
-                    value,
-                    expires,
-                )
+                with_conn(pool, async |conn| {
+                    Self::key_incr_(conn, key, value, expires).await
+                })
                 .await
             }
         }
@@ -76,16 +68,13 @@ impl RedisStore {
     pub async fn try_lock(&self, key: &[u8], expires: u64) -> trc::Result<bool> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.try_lock_(pool.get().await.map_err(into_error)?.as_mut(), key, expires)
-                    .await
+                with_conn(pool, async |conn| Self::try_lock_(conn, key, expires).await).await
             }
             RedisPool::Cluster(pool) => {
-                self.try_lock_(pool.get().await.map_err(into_error)?.as_mut(), key, expires)
-                    .await
+                with_conn(pool, async |conn| Self::try_lock_(conn, key, expires).await).await
             }
             RedisPool::Sentinel(pool) => {
-                self.try_lock_(pool.get().await.map_err(into_error)?.as_mut(), key, expires)
-                    .await
+                with_conn(pool, async |conn| Self::try_lock_(conn, key, expires).await).await
             }
         }
     }
@@ -93,16 +82,13 @@ impl RedisStore {
     pub async fn key_delete(&self, key: &[u8]) -> trc::Result<()> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.key_delete_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_delete_(conn, key).await).await
             }
             RedisPool::Cluster(pool) => {
-                self.key_delete_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_delete_(conn, key).await).await
             }
             RedisPool::Sentinel(pool) => {
-                self.key_delete_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_delete_(conn, key).await).await
             }
         }
     }
@@ -110,16 +96,22 @@ impl RedisStore {
     pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix)
-                    .await
+                with_conn(pool, async |conn| {
+                    Self::key_delete_prefix_(conn, prefix).await
+                })
+                .await
             }
             RedisPool::Cluster(pool) => {
-                self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix)
-                    .await
+                with_conn(pool, async |conn| {
+                    Self::key_delete_prefix_(conn, prefix).await
+                })
+                .await
             }
             RedisPool::Sentinel(pool) => {
-                self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix)
-                    .await
+                with_conn(pool, async |conn| {
+                    Self::key_delete_prefix_(conn, prefix).await
+                })
+                .await
             }
         }
     }
@@ -128,35 +120,31 @@ impl RedisStore {
         &self,
         key: &[u8],
     ) -> trc::Result<Option<T>> {
-        match &self.pool {
+        let value = match &self.pool {
             RedisPool::Single(pool) => {
-                self.key_get_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_get_(conn, key).await).await
             }
             RedisPool::Cluster(pool) => {
-                self.key_get_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_get_(conn, key).await).await
             }
             RedisPool::Sentinel(pool) => {
-                self.key_get_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_get_(conn, key).await).await
             }
-        }
+        }?;
+
+        value.map(T::deserialize_owned).transpose()
     }
 
     pub async fn counter_get(&self, key: &[u8]) -> trc::Result<i64> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.counter_get_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::counter_get_(conn, key).await).await
             }
             RedisPool::Cluster(pool) => {
-                self.counter_get_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::counter_get_(conn, key).await).await
             }
             RedisPool::Sentinel(pool) => {
-                self.counter_get_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::counter_get_(conn, key).await).await
             }
         }
     }
@@ -164,92 +152,69 @@ impl RedisStore {
     pub async fn key_exists(&self, key: &[u8]) -> trc::Result<bool> {
         match &self.pool {
             RedisPool::Single(pool) => {
-                self.key_exists_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_exists_(conn, key).await).await
             }
             RedisPool::Cluster(pool) => {
-                self.key_exists_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_exists_(conn, key).await).await
             }
             RedisPool::Sentinel(pool) => {
-                self.key_exists_(pool.get().await.map_err(into_error)?.as_mut(), key)
-                    .await
+                with_conn(pool, async |conn| Self::key_exists_(conn, key).await).await
             }
         }
     }
 
-    async fn key_get_<T: Deserialize + std::fmt::Debug + 'static>(
-        &self,
-        conn: &mut impl AsyncCommands,
-        key: &[u8],
-    ) -> trc::Result<Option<T>> {
-        if let Some(value) = redis::cmd("GET")
-            .arg(key)
-            .query_async::<Option<Vec<u8>>>(conn)
-            .await
-            .map_err(into_error)?
-        {
-            T::deserialize_owned(value).map(Some)
-        } else {
-            Ok(None)
-        }
+    async fn key_get_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<Option<Vec<u8>>> {
+        redis::cmd("GET").arg(key).query_async(conn).await
     }
 
-    async fn counter_get_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result<i64> {
+    async fn counter_get_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<i64> {
         redis::cmd("GET")
             .arg(key)
             .query_async::<Option<i64>>(conn)
             .await
-            .map(|x| x.unwrap_or(0))
-            .map_err(into_error)
+            .map(|value| value.unwrap_or(0))
     }
 
-    async fn key_exists_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result<bool> {
-        conn.exists(key).await.map_err(into_error)
+    async fn key_exists_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<bool> {
+        conn.exists(key).await
     }
 
     async fn key_set_(
-        &self,
         conn: &mut impl AsyncCommands,
         key: &[u8],
         value: &[u8],
         expires: Option<u64>,
-    ) -> trc::Result<()> {
+    ) -> RedisResult<()> {
         if let Some(expires) = expires {
-            conn.set_ex(key, value, expires).await.map_err(into_error)
+            conn.set_ex(key, value, expires).await
         } else {
-            conn.set(key, value).await.map_err(into_error)
+            conn.set(key, value).await
         }
     }
 
     async fn key_incr_(
-        &self,
         conn: &mut impl AsyncCommands,
         key: &[u8],
         value: i64,
         expires: Option<u64>,
-    ) -> trc::Result<i64> {
+    ) -> RedisResult<i64> {
         if let Some(expires) = expires {
-            redis::pipe()
-                .atomic()
-                .incr(key, value)
-                .expire(key, expires as i64)
-                .ignore()
-                .query_async::<Vec<i64>>(conn)
+            INCR_EXPIRE
+                .key(key)
+                .arg(value)
+                .arg(expires as i64)
+                .invoke_async(conn)
                 .await
-                .map_err(into_error)
-                .map(|v| v.first().copied().unwrap_or(0))
         } else {
-            conn.incr(key, value).await.map_err(into_error)
+            conn.incr(key, value).await
         }
     }
 
     async fn try_lock_(
-        &self,
         conn: &mut impl AsyncCommands,
         key: &[u8],
         expires: u64,
-    ) -> trc::Result<bool> {
+    ) -> RedisResult<bool> {
         redis::cmd("SET")
             .arg(key)
             .arg(now() + expires)
@@ -259,18 +224,13 @@ impl RedisStore {
             .query_async::<Option<String>>(conn)
             .await
             .map(|reply| reply.is_some())
-            .map_err(into_error)
     }
 
-    async fn key_delete_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result<()> {
-        conn.del(key).await.map_err(into_error)
+    async fn key_delete_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<()> {
+        conn.del(key).await
     }
 
-    async fn key_delete_prefix_(
-        &self,
-        conn: &mut impl AsyncCommands,
-        prefix: &[u8],
-    ) -> trc::Result<()> {
+    async fn key_delete_prefix_(conn: &mut impl AsyncCommands, prefix: &[u8]) -> RedisResult<()> {
         let mut pattern = Vec::with_capacity(prefix.len() + 1);
         pattern.extend_from_slice(prefix);
         pattern.push(b'*');
@@ -284,11 +244,10 @@ impl RedisStore {
                 .arg("COUNT")
                 .arg(100)
                 .query_async(conn)
-                .await
-                .map_err(into_error)?;
+                .await?;
 
             if !keys.is_empty() {
-                conn.del::<_, ()>(&keys).await.map_err(into_error)?;
+                conn.del::<_, ()>(&keys).await?;
             }
 
             if new_cursor != 0 {
@@ -299,3 +258,34 @@ impl RedisStore {
         }
     }
 }
+
+async fn with_conn<M, T>(
+    pool: &Pool<M>,
+    operation: impl AsyncFnOnce(&mut M::Type) -> RedisResult<T>,
+) -> trc::Result<T>
+where
+    M: Manager<Error = trc::Error>,
+{
+    let mut conn = pool.get().await.map_err(into_error)?;
+
+    match operation(conn.as_mut()).await {
+        Ok(value) => Ok(value),
+        Err(err) => {
+            if is_stale_connection(&err) {
+                drop(Object::take(conn));
+            }
+            Err(into_error(err))
+        }
+    }
+}
+
+fn is_stale_connection(err: &RedisError) -> bool {
+    matches!(
+        err.retry_method(),
+        RetryMethod::Reconnect
+            | RetryMethod::ReconnectFromInitialConnections
+            | RetryMethod::RefreshSlotsAndRetry
+            | RetryMethod::MovedRedirect
+            | RetryMethod::AskRedirect
+    )
+}

crates/trc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "trc"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/trc/event-macro/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "event_macro"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [lib]

crates/types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "types"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "utils"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [dependencies]

crates/utils/proc-macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proc_macros"
-version = "0.16.22"
+version = "0.16.23"
 edition = "2024"
 
 [lib]