Merge upstream v0.16.23

Five conflicts, resolved:

- crates/common/src/auth/authentication.rs: upstream's get_directory_for_token
  and JwtClaims replace extract_jwt_domain; the per-domain directory code
  (DIR-1, DIR-5 to DIR-7) is kept, and the token lookup routes through it.
  The release's one new Enterprise snippet was the body of
  get_directory_for_issuer, which stays returning None: a token naming no
  address gets the server default, as DIR-2 specifies and as v0.16.22 did.
- crates/common/src/manager/application.rs: upstream's rewrite of the tests,
  with the temp directory names renamed again, and the 5(a) notice the
  name-purge change should have added.
- crates/common/src/network/mta.rs: both sides' imports.
- crates/main/Cargo.toml: the AGPL-only license kept, version 0.16.23.
- Cargo.lock: upstream's, with the fork's crates added by Cargo.

tests/src/smtp/lookup/expressions.rs

@@ -37,6 +37,10 @@ const TESTS: &[(&str, &str)] = &[
         "is_local_domain('FooBar.org') + '-' + is_local_address('[email protected]') + '-' + is_local_address('[email protected]')",
         "1-1-1",
     ),
+    (
+        "bit_and(254, 16) + '-' + bit_and(254, 1) + '-' + bit_and(80, 64) + '-' + bit_and(255, 128)",
+        "16-0-64-128",
+    ),
 ];
 
 #[tokio::test]

tests/src/smtp/outbound/fallback_relay.rs

@@ -127,7 +127,9 @@ async fn fallback_relay() {
     let next_due = now();
     let queue_id = retry.queue_id;
     retry.message.recipients[0].retry.due = next_due;
-    retry.save_changes(&local.server, prev_due.into()).await;
+    retry
+        .save_changes(&local.server, prev_due.into(), None)
+        .await;
     local
         .delivery_attempt(queue_id)
         .await

tests/src/smtp/outbound/tls.rs

@@ -144,7 +144,9 @@ async fn starttls_optional() {
     let next_due = now();
     let queue_id = retry.queue_id;
     retry.message.recipients[0].retry.due = next_due;
-    retry.save_changes(&local.server, prev_due.into()).await;
+    retry
+        .save_changes(&local.server, prev_due.into(), None)
+        .await;
     local
         .delivery_attempt_for_queue(queue_id, "default")
         .await

tests/src/smtp/queue/dsn.rs

@@ -8,11 +8,12 @@ use crate::utils::server::{TestServer, TestServerBuilder};
 use common::config::smtp::queue::{QueueExpiry, QueueName};
 use registry::schema::{
     enums::CompressionAlgo,
-    structs::{DsnReportSettings, Expression, ReportSettings},
+    structs::{DsnReportSettings, Expression, FileSystemStore, ReportSettings},
 };
 use smtp::queue::{
-    Error, ErrorDetails, HostResponse, Message, MessageWrapper, Recipient, Schedule, Status,
-    UnexpectedResponse, dsn::SendDsn,
+    Error, ErrorDetails, HostResponse, Message, MessageWrapper, RCPT_DSN_SENT, Recipient, Schedule,
+    Status, UnexpectedResponse,
+    dsn::{DsnStatus, SendDsn},
 };
 use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS, Response};
 use std::{
@@ -21,7 +22,7 @@ use std::{
     path::PathBuf,
     time::SystemTime,
 };
-use store::write::now;
+use store::{BlobStore, backend::fs::FsStore, write::now};
 use types::blob_hash::BlobHash;
 
 #[tokio::test]
@@ -194,9 +195,65 @@ async fn generate_dsn() {
     // Load queue
     let queue = local.read_queued_messages().await;
     assert_eq!(queue.len(), 4);
+
+    // A DSN that cannot be written is retried rather than marked as sent
+    message.message.recipients = vec![Recipient {
+        address: "[email protected]".into(),
+        status: Status::PermanentFailure(ErrorDetails {
+            entity: "mx.example.org".into(),
+            details: Error::UnexpectedResponse(UnexpectedResponse {
+                command: "RCPT TO:<[email protected]>".into(),
+                response: Response {
+                    code: 550,
+                    esc: [5, 1, 2],
+                    message: "User does not exist".into(),
+                },
+            }),
+        }),
+        flags,
+        orcpt: None,
+        retry: Schedule::now(),
+        notify: Schedule::now(),
+        expires: QueueExpiry::Ttl(10),
+        queue: QueueName::default(),
+    }];
+
+    let blob_store = local.server.blob_store().clone();
+    local.set_blob_store(unwritable_blob_store(local.tmp_dir()).await);
+    assert_eq!(
+        local.server.send_dsn(&mut message).await,
+        DsnStatus::Deferred
+    );
+    assert_eq!(message.message.recipients[0].flags & RCPT_DSN_SENT, 0);
+    local.assert_no_events();
+    assert_eq!(local.read_queued_messages().await.len(), 4);
+
+    local.set_blob_store(blob_store);
+    assert_eq!(
+        local.server.send_dsn(&mut message).await,
+        DsnStatus::Completed
+    );
+    assert_ne!(message.message.recipients[0].flags & RCPT_DSN_SENT, 0);
+    local.expect_message().await;
+    assert_eq!(local.read_queued_messages().await.len(), 5);
+}
+
+async fn unwritable_blob_store(tmp_dir: &str) -> BlobStore {
+    let path = format!("{tmp_dir}/unwritable-blob-store");
+    fs::write(&path, b"").unwrap();
+
+    FsStore::open(FileSystemStore { path, depth: 0 })
+        .await
+        .unwrap()
 }
 
 impl TestServer {
+    fn set_blob_store(&mut self, blob: BlobStore) {
+        let mut core = self.server.core.as_ref().clone();
+        core.storage.blob = blob;
+        self.server.core = core.into();
+    }
+
     async fn compare_dsn(&self, message: Message, test: &str) {
         let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
         path.push("resources");

tests/src/smtp/queue/manager.rs

@@ -33,15 +33,15 @@ async fn queue_due() {
 
     let mut message = new_message(0);
     message.message.recipients.push(build_rcpt("c", 3, 8, 9));
-    message.save_changes(&local.server, 0.into()).await;
+    message.save_changes(&local.server, 0.into(), None).await;
 
     let mut message = new_message(1);
     message.message.recipients.push(build_rcpt("b", 2, 6, 7));
-    message.save_changes(&local.server, 0.into()).await;
+    message.save_changes(&local.server, 0.into(), None).await;
 
     let mut message = new_message(2);
     message.message.recipients.push(build_rcpt("a", 1, 4, 5));
-    message.save_changes(&local.server, 0.into()).await;
+    message.save_changes(&local.server, 0.into(), None).await;
 
     for domain in vec!["a", "b", "c"].into_iter() {
         let now = now();