Import upstream v0.16.23, stripped

Upstream commit: 9d1c75ab68435e4417337f768291e5f947686203
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Verification: clean

One snippet more than v0.16.22, in crates/common/src/auth/authentication.rs
(3, was 2).

tests/src/directory/issuer.rs

@@ -0,0 +1,115 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    directory::oidc::get_token_for_client,
+    utils::{containers, server::TestServerBuilder},
+};
+use base64::{Engine, engine::general_purpose};
+use registry::{
+    schema::{
+        prelude::{ObjectType, Property},
+        structs::{Action, Directory, OidcDirectory},
+    },
+    types::map::Map,
+};
+use serde_json::json;
+use std::time::Duration;
+
+const ISSUER: &str = "http://localhost:9080/realms/stalwart";
+const DOMAIN: &str = "example.org";
+const ACCOUNT: &str = "[email protected]";
+const PASSWORD: &str = "this is an OIDC password";
+
+pub async fn test() {
+    println!("Running OIDC issuer routing tests...");
+    containers::ensure_keycloak().await;
+    let test = TestServerBuilder::new("directory_issuer_test")
+        .await
+        .with_default_listeners()
+        .await
+        .disable_services()
+        .build()
+        .await;
+
+    let admin = test.account("admin");
+    let directory_id = admin
+        .registry_create_object(Directory::Oidc(OidcDirectory {
+            description: "Issuer routing test OIDC directory".to_string(),
+            issuer_url: ISSUER.to_string(),
+            claim_username: "email".to_string(),
+            claim_name: Some("name".to_string()),
+            claim_groups: Some("groups".to_string()),
+            require_audience: Some("stalwart".to_string()),
+            require_scopes: Map::new(vec!["openid".to_string()]),
+            ..Default::default()
+        }))
+        .await;
+    let domain_id = admin.find_or_create_domain(DOMAIN).await;
+    admin
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({ Property::DirectoryId: directory_id.to_string() }),
+        )
+        .await;
+    admin.reload_settings().await;
+    admin.registry_create_object(Action::InvalidateCaches).await;
+
+    assert!(
+        test.server.get_default_directory().is_none(),
+        "The OIDC directory must be reachable only through the domain for this test to mean anything"
+    );
+
+    let token = get_token_for_client(
+        "stalwart-fallback",
+        "stalwart-fallback-secret",
+        ACCOUNT,
+        PASSWORD,
+        "openid",
+    )
+    .await;
+    let claims = access_token_claims(&token);
+    for claim in ["email", "preferred_username", "upn"] {
+        assert!(
+            claims.get(claim).is_none(),
+            "The access token carries a {claim} claim, so it no longer covers issuer based routing: {claims}"
+        );
+    }
+    assert_eq!(claims["iss"], json!(ISSUER));
+
+    assert_eq!(
+        session_status(&token).await,
+        200,
+        "A bearer token without a username claim did not reach the domain's OIDC directory"
+    );
+}
+
+fn access_token_claims(token: &str) -> serde_json::Value {
+    let payload = token.split('.').nth(1).expect("The token is not a JWT");
+
+    serde_json::from_slice(
+        &general_purpose::URL_SAFE_NO_PAD
+            .decode(payload)
+            .expect("Failed to decode the token payload"),
+    )
+    .expect("Failed to parse the token claims")
+}
+
+async fn session_status(token: &str) -> u16 {
+    reqwest::Client::builder()
+        .timeout(Duration::from_secs(30))
+        .danger_accept_invalid_certs(true)
+        .build()
+        .unwrap()
+        .get("https://127.0.0.1:8899/jmap/session")
+        .bearer_auth(token)
+        .send()
+        .await
+        .expect("Failed to send session request")
+        .status()
+        .as_u16()
+}

tests/src/directory/mod.rs

@@ -6,6 +6,7 @@
 
 pub mod discovery;
 pub mod integration;
+pub mod issuer;
 pub mod ldap;
 #[cfg(feature = "sqlite")]
 pub mod sql;
@@ -18,6 +19,7 @@ pub async fn directory_tests() {
     oidc::test().await;
     unavailable::test().await;
     discovery::test().await;
+    issuer::test().await;
     #[cfg(feature = "sqlite")]
     sql::test().await;
     synchronization::test().await;

tests/src/imap/antispam.rs

@@ -133,6 +133,41 @@ pub async fn test(test: &TestServer) {
     let samples = account.spam_training_samples().await;
     assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 11);
     assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 10);
+
+    let support_id = test.account("[email protected]").id();
+    let jane = test.account("[email protected]");
+    let jane_id = jane.id();
+    let mut imap_jane = jane.imap_client().await;
+    let samples_for = |account_id, is_spam: bool| {
+        let admin = &admin;
+        async move {
+            admin
+                .spam_training_samples()
+                .await
+                .into_iter()
+                .filter(|(_, sample)| {
+                    sample.account_id == Some(account_id) && sample.is_spam == is_spam
+                })
+                .count()
+        }
+    };
+
+    imap_jane
+        .append("Shared Folders/[email protected]/Drafts", SPAM[1])
+        .await;
+    assert_eq!(samples_for(support_id, true).await, 0);
+
+    imap_jane
+        .send_ok("SELECT \"Shared Folders/[email protected]/Drafts\"")
+        .await;
+    imap_jane.send_ok("MOVE * \"Junk Mail\"").await;
+    assert_eq!(samples_for(support_id, true).await, 1);
+
+    imap_jane.send_ok("SELECT \"Junk Mail\"").await;
+    imap_jane
+        .send_ok("MOVE * \"Shared Folders/[email protected]/Drafts\"")
+        .await;
+    assert_eq!(samples_for(jane_id, false).await, 1);
 }
 
 pub async fn spam_classifier_model(server: &Server) -> SpamTrainer {

tests/src/imap/pop.rs

@@ -122,18 +122,32 @@ pub async fn test(test: &TestServer) {
     pop3.assert_read(ResponseType::Err).await;
 
     // TOP
-    pop3.send("TOP 1 4").await;
+    pop3.send("TOP 1 0").await;
     pop3.assert_read(ResponseType::Multiline)
         .await
-        .assert_contains("+OK 203 octets")
         .assert_contains("Subject: TPS Report 0")
+        .assert_contains("X-Spam-Status: No")
         .assert_not_contains("I'm going to need those TPS 0 reports ASAP.");
-    pop3.send("TOP 3 4").await;
+    pop3.send("TOP 1 1").await;
+    pop3.assert_read(ResponseType::Multiline)
+        .await
+        .assert_contains("Subject: TPS Report 0")
+        .assert_contains("I'm going to need those TPS 0 reports ASAP.")
+        .assert_not_contains("So, if you could do that, that'd be great.");
+    pop3.send("TOP 3 0").await;
+    pop3.assert_read(ResponseType::Multiline)
+        .await
+        .assert_contains("Subject: TPS Report 2")
+        .assert_not_contains("I'm going to need those TPS 2 reports ASAP.");
+    pop3.send("TOP 3 100").await;
     pop3.assert_read(ResponseType::Multiline)
         .await
         .assert_contains("+OK 203 octets")
         .assert_contains("Subject: TPS Report 2")
-        .assert_not_contains("I'm going to need those TPS 2 reports ASAP.");
+        .assert_contains("I'm going to need those TPS 2 reports ASAP.")
+        .assert_contains("So, if you could do that, that'd be great.");
+    pop3.send("TOP 4 1").await;
+    pop3.assert_read(ResponseType::Err).await;
 
     // DELE + RSET + QUIT (should not delete messages)
     pop3.send("DELE 1").await;

tests/src/smtp/lookup/expressions.rs

@@ -37,6 +37,10 @@ const TESTS: &[(&str, &str)] = &[
         "is_local_domain('FooBar.org') + '-' + is_local_address('[email protected]') + '-' + is_local_address('[email protected]')",
         "1-1-1",
     ),
+    (
+        "bit_and(254, 16) + '-' + bit_and(254, 1) + '-' + bit_and(80, 64) + '-' + bit_and(255, 128)",
+        "16-0-64-128",
+    ),
 ];
 
 #[tokio::test]

tests/src/smtp/outbound/fallback_relay.rs

@@ -127,7 +127,9 @@ async fn fallback_relay() {
     let next_due = now();
     let queue_id = retry.queue_id;
     retry.message.recipients[0].retry.due = next_due;
-    retry.save_changes(&local.server, prev_due.into()).await;
+    retry
+        .save_changes(&local.server, prev_due.into(), None)
+        .await;
     local
         .delivery_attempt(queue_id)
         .await

tests/src/smtp/outbound/tls.rs

@@ -144,7 +144,9 @@ async fn starttls_optional() {
     let next_due = now();
     let queue_id = retry.queue_id;
     retry.message.recipients[0].retry.due = next_due;
-    retry.save_changes(&local.server, prev_due.into()).await;
+    retry
+        .save_changes(&local.server, prev_due.into(), None)
+        .await;
     local
         .delivery_attempt_for_queue(queue_id, "default")
         .await

tests/src/smtp/queue/dsn.rs

@@ -8,11 +8,12 @@ use crate::utils::server::{TestServer, TestServerBuilder};
 use common::config::smtp::queue::{QueueExpiry, QueueName};
 use registry::schema::{
     enums::CompressionAlgo,
-    structs::{DsnReportSettings, Expression, ReportSettings},
+    structs::{DsnReportSettings, Expression, FileSystemStore, ReportSettings},
 };
 use smtp::queue::{
-    Error, ErrorDetails, HostResponse, Message, MessageWrapper, Recipient, Schedule, Status,
-    UnexpectedResponse, dsn::SendDsn,
+    Error, ErrorDetails, HostResponse, Message, MessageWrapper, RCPT_DSN_SENT, Recipient, Schedule,
+    Status, UnexpectedResponse,
+    dsn::{DsnStatus, SendDsn},
 };
 use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS, Response};
 use std::{
@@ -21,7 +22,7 @@ use std::{
     path::PathBuf,
     time::SystemTime,
 };
-use store::write::now;
+use store::{BlobStore, backend::fs::FsStore, write::now};
 use types::blob_hash::BlobHash;
 
 #[tokio::test]
@@ -194,9 +195,65 @@ async fn generate_dsn() {
     // Load queue
     let queue = local.read_queued_messages().await;
     assert_eq!(queue.len(), 4);
+
+    // A DSN that cannot be written is retried rather than marked as sent
+    message.message.recipients = vec![Recipient {
+        address: "[email protected]".into(),
+        status: Status::PermanentFailure(ErrorDetails {
+            entity: "mx.example.org".into(),
+            details: Error::UnexpectedResponse(UnexpectedResponse {
+                command: "RCPT TO:<[email protected]>".into(),
+                response: Response {
+                    code: 550,
+                    esc: [5, 1, 2],
+                    message: "User does not exist".into(),
+                },
+            }),
+        }),
+        flags,
+        orcpt: None,
+        retry: Schedule::now(),
+        notify: Schedule::now(),
+        expires: QueueExpiry::Ttl(10),
+        queue: QueueName::default(),
+    }];
+
+    let blob_store = local.server.blob_store().clone();
+    local.set_blob_store(unwritable_blob_store(local.tmp_dir()).await);
+    assert_eq!(
+        local.server.send_dsn(&mut message).await,
+        DsnStatus::Deferred
+    );
+    assert_eq!(message.message.recipients[0].flags & RCPT_DSN_SENT, 0);
+    local.assert_no_events();
+    assert_eq!(local.read_queued_messages().await.len(), 4);
+
+    local.set_blob_store(blob_store);
+    assert_eq!(
+        local.server.send_dsn(&mut message).await,
+        DsnStatus::Completed
+    );
+    assert_ne!(message.message.recipients[0].flags & RCPT_DSN_SENT, 0);
+    local.expect_message().await;
+    assert_eq!(local.read_queued_messages().await.len(), 5);
+}
+
+async fn unwritable_blob_store(tmp_dir: &str) -> BlobStore {
+    let path = format!("{tmp_dir}/unwritable-blob-store");
+    fs::write(&path, b"").unwrap();
+
+    FsStore::open(FileSystemStore { path, depth: 0 })
+        .await
+        .unwrap()
 }
 
 impl TestServer {
+    fn set_blob_store(&mut self, blob: BlobStore) {
+        let mut core = self.server.core.as_ref().clone();
+        core.storage.blob = blob;
+        self.server.core = core.into();
+    }
+
     async fn compare_dsn(&self, message: Message, test: &str) {
         let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
         path.push("resources");

tests/src/smtp/queue/manager.rs

@@ -33,15 +33,15 @@ async fn queue_due() {
 
     let mut message = new_message(0);
     message.message.recipients.push(build_rcpt("c", 3, 8, 9));
-    message.save_changes(&local.server, 0.into()).await;
+    message.save_changes(&local.server, 0.into(), None).await;
 
     let mut message = new_message(1);
     message.message.recipients.push(build_rcpt("b", 2, 6, 7));
-    message.save_changes(&local.server, 0.into()).await;
+    message.save_changes(&local.server, 0.into(), None).await;
 
     let mut message = new_message(2);
     message.message.recipients.push(build_rcpt("a", 1, 4, 5));
-    message.save_changes(&local.server, 0.into()).await;
+    message.save_changes(&local.server, 0.into(), None).await;
 
     for domain in vec!["a", "b", "c"].into_iter() {
         let now = now();

tests/src/system/delivery.rs

@@ -587,14 +587,7 @@ END:VCARD
         })
         .await;
 
-    let bill_messages = test
-        .server
-        .get_cached_messages(bill.id().document_id())
-        .await
-        .unwrap()
-        .emails
-        .items
-        .len();
+    let bill_messages = inbox_count(&test.server, &bill).await;
 
     lmtp.ingest(
         "[email protected]",
@@ -612,17 +605,122 @@ END:VCARD
     tokio::time::sleep(Duration::from_millis(200)).await;
 
     assert_eq!(
-        test.server
-            .get_cached_messages(bill.id().document_id())
-            .await
-            .unwrap()
-            .emails
-            .items
-            .len(),
+        inbox_count(&test.server, &bill).await,
         bill_messages + 1,
         "sub-addressed mailing list member was not delivered"
     );
 
+    // Lists nested within lists must be expanded recursively
+    admin
+        .registry_create_object(MailingList {
+            name: "engineering".to_string(),
+            recipients: Map::new(vec!["[email protected]".to_string()]),
+            domain_id,
+            ..Default::default()
+        })
+        .await;
+    admin
+        .registry_create_object(MailingList {
+            name: "all-staff".to_string(),
+            recipients: Map::new(vec![
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+            ]),
+            domain_id,
+            ..Default::default()
+        })
+        .await;
+
+    let jane_messages = inbox_count(&test.server, &jane).await;
+    let bill_messages = inbox_count(&test.server, &bill).await;
+
+    lmtp.ingest(
+        "[email protected]",
+        &["[email protected]"],
+        concat!(
+            "From: [email protected]\r\n",
+            "To: [email protected]\r\n",
+            "Subject: Company picnic\r\n",
+            "\r\n",
+            "Bring your own stapler."
+        ),
+    )
+    .await;
+
+    tokio::time::sleep(Duration::from_millis(200)).await;
+
+    assert_eq!(
+        inbox_count(&test.server, &jane).await,
+        jane_messages + 1,
+        "nested mailing list member was not delivered"
+    );
+    assert_eq!(
+        inbox_count(&test.server, &bill).await,
+        bill_messages + 1,
+        "direct mailing list member was not delivered"
+    );
+
+    // Lists that reference each other must terminate and deliver exactly once
+    admin
+        .registry_create_object(MailingList {
+            name: "ouroboros-head".to_string(),
+            recipients: Map::new(vec![
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+            ]),
+            domain_id,
+            ..Default::default()
+        })
+        .await;
+    admin
+        .registry_create_object(MailingList {
+            name: "ouroboros-tail".to_string(),
+            recipients: Map::new(vec![
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+            ]),
+            domain_id,
+            ..Default::default()
+        })
+        .await;
+
+    let john_messages = inbox_count(&test.server, &john).await;
+    let jane_messages = inbox_count(&test.server, &jane).await;
+    let bill_messages = inbox_count(&test.server, &bill).await;
+
+    lmtp.ingest(
+        "[email protected]",
+        &["[email protected]"],
+        concat!(
+            "From: [email protected]\r\n",
+            "To: [email protected]\r\n",
+            "Subject: Going in circles\r\n",
+            "\r\n",
+            "Please advise."
+        ),
+    )
+    .await;
+
+    tokio::time::sleep(Duration::from_millis(200)).await;
+
+    assert_eq!(
+        inbox_count(&test.server, &john).await,
+        john_messages + 1,
+        "cyclic mailing list did not deliver exactly once"
+    );
+    assert_eq!(
+        inbox_count(&test.server, &jane).await,
+        jane_messages + 1,
+        "cyclic mailing list did not deliver exactly once"
+    );
+    assert_eq!(
+        inbox_count(&test.server, &bill).await,
+        bill_messages + 1,
+        "member shared by two nested lists was delivered more than once"
+    );
+
     // Remove test data
     john.registry_destroy(
         ObjectType::MaskedEmail,
@@ -703,3 +801,13 @@ async fn message_metadata(server: &Server, account_id: u32, document_id: u32) ->
         .deserialize::<MessageMetadata>()
         .unwrap()
 }
+
+async fn inbox_count(server: &Server, account: &Account) -> usize {
+    server
+        .get_cached_messages(account.id().document_id())
+        .await
+        .unwrap()
+        .emails
+        .items
+        .len()
+}