Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

tests/src/utils/account.rs

@@ -0,0 +1,299 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::utils::{imap::ImapConnection, server::TestServer, webdav::DummyWebDavClient};
+use ahash::AHashMap;
+use jmap_client::client::{Client, Credentials};
+use registry::{
+    schema::{
+        enums::Permission,
+        prelude::{ObjectType, Property},
+        structs::{
+            self, CertificateManagement, Credential, CustomRoles, DkimManagement, DnsManagement,
+            Domain, EmailAlias, GroupAccount, PasswordCredential, Permissions, PermissionsList,
+            Roles, UserAccount,
+        },
+    },
+    types::{list::List, map::Map},
+};
+use serde_json::json;
+use std::time::Duration;
+use types::id::Id;
+
+pub struct Account {
+    name: &'static str,
+    secret: &'static str,
+    emails: &'static [&'static str],
+    description: &'static str,
+    id: Id,
+    id_string: String,
+    pub http_listener_port: u16,
+}
+
+impl TestServer {
+    pub async fn create_user_account(
+        &self,
+        using_account: &str,
+        name: &'static str,
+        secret: &'static str,
+        aliases: &'static [&'static str],
+        description: &'static str,
+    ) -> Account {
+        self.account(using_account)
+            .create_user_account(name, secret, description, aliases, vec![])
+            .await
+    }
+
+    pub async fn create_admin_account(&self, name: &'static str) -> Account {
+        let admin = self
+            .create_user_account(
+                "admin",
+                name,
+                "these_pretzels_are_making_me_thirsty",
+                &[],
+                "Admin",
+            )
+            .await;
+        self.account("admin")
+            .assign_roles_to_account(admin.id(), &["user", "system"])
+            .await;
+        admin
+    }
+
+    pub fn insert_account(&mut self, account: Account) {
+        self.accounts.insert(account.name(), account);
+    }
+}
+
+impl Account {
+    pub fn new(
+        name: &'static str,
+        secret: &'static str,
+        emails: &'static [&'static str],
+        description: &'static str,
+        id: Id,
+    ) -> Self {
+        Self {
+            name,
+            secret,
+            emails,
+            description,
+            id,
+            id_string: id.to_string(),
+            http_listener_port: 8899,
+        }
+    }
+
+    pub fn update_secret(&mut self, new_secret: &'static str) {
+        self.secret = new_secret;
+    }
+
+    pub fn id(&self) -> Id {
+        self.id
+    }
+
+    pub fn id_string(&self) -> &str {
+        &self.id_string
+    }
+
+    pub fn name(&self) -> &'static str {
+        self.name
+    }
+
+    pub fn description(&self) -> &'static str {
+        self.description
+    }
+
+    pub fn secret(&self) -> &'static str {
+        self.secret
+    }
+
+    pub fn emails(&self) -> &'static [&'static str] {
+        self.emails
+    }
+
+    pub async fn find_or_create_domain(&self, name: &'static str) -> Id {
+        let ids = self
+            .registry_query_ids(
+                ObjectType::Domain,
+                [(Property::Name, name)],
+                Vec::<&str>::new(),
+            )
+            .await;
+
+        match ids.len() {
+            0 => self.create_domain(name).await,
+            1 => ids[0],
+            _ => panic!("Multiple domains with name {name} found"),
+        }
+    }
+
+    pub async fn create_user_account(
+        &self,
+        name: &'static str,
+        secret: &'static str,
+        description: &'static str,
+        aliases: &'static [&'static str],
+        extra_permissions: Vec<Permission>,
+    ) -> Account {
+        let mut domains = AHashMap::from_iter(aliases.iter().copied().chain([name]).map(|email| {
+            let domain = email.split('@').nth(1).expect("Invalid email address");
+            (domain, Id::singleton())
+        }));
+        for (name, id) in &mut domains {
+            *id = self.find_or_create_domain(name).await;
+        }
+        let (account_name, domain_id) = name
+            .rsplit_once('@')
+            .map(|(name, domain)| (name.to_string(), *domains.get(domain).unwrap()))
+            .unwrap();
+        let account_aliases = aliases.iter().filter(|email| **email != name).map(|email| {
+            let (name, domain_id) = email
+                .rsplit_once('@')
+                .map(|(name, domain)| (name.to_string(), *domains.get(domain).unwrap()))
+                .unwrap();
+            EmailAlias {
+                name,
+                domain_id,
+                enabled: true,
+                ..Default::default()
+            }
+        });
+
+        let account_id = self
+            .registry_create_object(structs::Account::User(UserAccount {
+                name: account_name,
+                domain_id,
+                credentials: List::from_iter([Credential::Password(PasswordCredential {
+                    secret: secret.to_string(),
+                    ..Default::default()
+                })]),
+                aliases: List::from_iter(account_aliases),
+                description: description.to_string().into(),
+                permissions: Permissions::Merge(PermissionsList {
+                    disabled_permissions: Default::default(),
+                    enabled_permissions: Map::new(extra_permissions),
+                }),
+                ..Default::default()
+            }))
+            .await;
+
+        let mut account = Account::new(name, secret, aliases, description, account_id);
+        account.http_listener_port = self.http_listener_port;
+        account
+    }
+
+    pub async fn create_group_account(
+        &self,
+        name: &'static str,
+        description: &'static str,
+        aliases: &'static [&'static str],
+    ) -> Account {
+        let mut domains = AHashMap::from_iter(aliases.iter().copied().chain([name]).map(|email| {
+            let domain = email.split('@').nth(1).expect("Invalid email address");
+            (domain, Id::singleton())
+        }));
+        for (name, id) in &mut domains {
+            *id = self.find_or_create_domain(name).await;
+        }
+        let (account_name, domain_id) = name
+            .rsplit_once('@')
+            .map(|(name, domain)| (name.to_string(), *domains.get(domain).unwrap()))
+            .unwrap();
+        let account_aliases = aliases.iter().map(|email| {
+            let (name, domain_id) = email
+                .rsplit_once('@')
+                .map(|(name, domain)| (name.to_string(), *domains.get(domain).unwrap()))
+                .unwrap();
+            EmailAlias {
+                name,
+                domain_id,
+                enabled: true,
+                ..Default::default()
+            }
+        });
+
+        let account_id = self
+            .registry_create_object(structs::Account::Group(GroupAccount {
+                name: account_name,
+                domain_id,
+                aliases: List::from_iter(account_aliases),
+                description: description.to_string().into(),
+                ..Default::default()
+            }))
+            .await;
+
+        Account::new(name, "", aliases, description, account_id)
+    }
+
+    pub async fn create_domain(&self, name: &'static str) -> Id {
+        self.registry_create_object(Domain {
+            is_enabled: true,
+            name: name.to_string(),
+            certificate_management: CertificateManagement::Manual,
+            dns_management: DnsManagement::Manual,
+            dkim_management: DkimManagement::Manual,
+            ..Default::default()
+        })
+        .await
+    }
+
+    pub async fn assign_roles_to_account(&self, account_id: Id, names: &[&str]) {
+        let mut role_ids = Vec::new();
+        for name in names {
+            let role_id = *self
+                .registry_query_ids(
+                    ObjectType::Role,
+                    [(Property::Description, *name)],
+                    Vec::<&str>::new(),
+                )
+                .await
+                .first()
+                .unwrap_or_else(|| panic!("Role {name} not found"));
+            role_ids.push(role_id);
+        }
+
+        self.registry_update(
+            ObjectType::Account,
+            [(
+                account_id,
+                json!({
+                    Property::Roles: Roles::Custom(CustomRoles { role_ids: Map::new(role_ids) })
+                }),
+            )],
+        )
+        .await
+        .updated_id(account_id);
+    }
+
+    pub fn webdav_client(&self) -> DummyWebDavClient {
+        DummyWebDavClient::new(
+            self.id.document_id(),
+            self.name(),
+            self.secret(),
+            self.emails()[0],
+        )
+    }
+
+    pub async fn imap_client(&self) -> ImapConnection {
+        let mut imap = ImapConnection::connect(b"_x ").await;
+        imap.authenticate(self.name(), self.secret()).await;
+        imap
+    }
+
+    pub async fn jmap_client(&self) -> Client {
+        let mut client = Client::new()
+            .credentials(Credentials::basic(self.name(), self.secret()))
+            .timeout(Duration::from_secs(3600))
+            .accept_invalid_certs(true)
+            .follow_redirects(["127.0.0.1"])
+            .connect(&format!("https://127.0.0.1:{}", self.http_listener_port))
+            .await
+            .unwrap();
+        client.set_default_account_id(self.id_string());
+        client
+    }
+}

tests/src/utils/cleanup.rs

@@ -0,0 +1,428 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use ::registry::{
+    schema::prelude::{OBJ_SINGLETON, ObjectType},
+    types::EnumImpl,
+};
+use store::{
+    ValueKey,
+    write::{key::DeserializeBigEndian, *},
+    *,
+};
+use trc::AddContext;
+use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
+
+pub async fn store_destroy(store: &Store) {
+    store_destroy_sql_indexes(store).await;
+
+    for subspace in [
+        SUBSPACE_ACL,
+        SUBSPACE_TASK_QUEUE,
+        SUBSPACE_INDEXES,
+        SUBSPACE_DELETED_ITEMS,
+        SUBSPACE_SPAM_SAMPLES,
+        SUBSPACE_BLOB_LINK,
+        SUBSPACE_LOGS,
+        SUBSPACE_IN_MEMORY_COUNTER,
+        SUBSPACE_IN_MEMORY_VALUE,
+        SUBSPACE_COUNTER,
+        SUBSPACE_PROPERTY,
+        SUBSPACE_REGISTRY,
+        SUBSPACE_BLOBS,
+        SUBSPACE_QUEUE_MESSAGE,
+        SUBSPACE_QUEUE_EVENT,
+        SUBSPACE_QUOTA,
+        SUBSPACE_REPORT_OUT,
+        SUBSPACE_REPORT_IN,
+        SUBSPACE_TELEMETRY_SPAN,
+        SUBSPACE_TELEMETRY_METRIC,
+        SUBSPACE_SEARCH_INDEX,
+        SUBSPACE_REGISTRY_IDX,
+        SUBSPACE_REGISTRY_PK,
+        SUBSPACE_DIRECTORY,
+    ] {
+        if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() {
+            continue;
+        }
+
+        store
+            .delete_range(
+                AnyKey {
+                    subspace,
+                    key: vec![0u8],
+                },
+                AnyKey {
+                    subspace,
+                    key: vec![u8::MAX; 16],
+                },
+            )
+            .await
+            .unwrap();
+    }
+}
+
+pub async fn search_store_destroy(store: &SearchStore) {
+    match &store {
+        SearchStore::Store(store) => {
+            store_destroy_sql_indexes(store).await;
+        }
+        SearchStore::ElasticSearch(store) => {
+            if let Err(err) = store.drop_indexes().await {
+                eprintln!("Failed to drop elasticsearch indexes: {}", err);
+            }
+            store.create_indexes().await.unwrap();
+        }
+        SearchStore::MeiliSearch(store) => {
+            if let Err(err) = store.drop_indexes().await {
+                eprintln!("Failed to drop meilisearch indexes: {}", err);
+            }
+            store.create_indexes().await.unwrap();
+        }
+    }
+}
+
+#[allow(unused_variables)]
+async fn store_destroy_sql_indexes(store: &Store) {
+    #[cfg(any(feature = "postgres", feature = "mysql"))]
+    {
+        if store.is_pg_or_mysql() {
+            for index in [
+                SearchIndex::Email,
+                SearchIndex::Calendar,
+                SearchIndex::Contacts,
+                SearchIndex::Tracing,
+            ] {
+                #[cfg(feature = "postgres")]
+                let table = index.psql_table();
+                #[cfg(feature = "mysql")]
+                let table = index.mysql_table();
+
+                let _ = store
+                    .sql_query::<usize>(&format!("TRUNCATE TABLE {table}"), vec![])
+                    .await;
+            }
+        }
+    }
+}
+
+pub async fn store_blob_expire_all(store: &Store) {
+    // Delete all temporary hashes
+    let from_key = ValueKey {
+        account_id: 0,
+        collection: 0,
+        document_id: 0,
+        class: ValueClass::Blob(BlobOp::Commit {
+            hash: BlobHash::default(),
+        }),
+    };
+    let to_key = ValueKey {
+        account_id: u32::MAX,
+        collection: u8::MAX,
+        document_id: u32::MAX,
+        class: ValueClass::Blob(BlobOp::Link {
+            hash: BlobHash::new_max(),
+            to: BlobLink::Document,
+        }),
+    };
+    let mut batch = BatchBuilder::new();
+    let mut last_account_id = u32::MAX;
+    store
+        .iterate(
+            IterateParams::new(from_key, to_key).ascending(),
+            |key, _| {
+                if key.len() == BLOB_HASH_LEN + U32_LEN + U64_LEN {
+                    let account_id = key
+                        .deserialize_be_u32(BLOB_HASH_LEN)
+                        .caused_by(trc::location!())?;
+                    if account_id != last_account_id {
+                        last_account_id = account_id;
+                        batch.with_account_id(account_id);
+                    }
+                    let hash =
+                        BlobHash::try_from_hash_slice(key.get(..BLOB_HASH_LEN).unwrap()).unwrap();
+                    let until = key
+                        .deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)
+                        .caused_by(trc::location!())?;
+
+                    batch.clear(ValueClass::Blob(BlobOp::Link {
+                        hash,
+                        to: BlobLink::Temporary { until },
+                    }));
+                }
+
+                Ok(true)
+            },
+        )
+        .await
+        .unwrap();
+    store.write(batch.build_all()).await.unwrap();
+}
+
+pub async fn store_lookup_expire_all(store: &Store) {
+    // Delete all temporary counters
+    let from_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![0u8])));
+    let to_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![u8::MAX; 10])));
+
+    let mut expired_keys = Vec::new();
+    let mut expired_counters = Vec::new();
+
+    store
+        .iterate(IterateParams::new(from_key, to_key), |key, value| {
+            let expiry = value.deserialize_be_u64(0).caused_by(trc::location!())?;
+            if expiry == 0 {
+                expired_counters.push(key.to_vec());
+            } else if expiry != u64::MAX {
+                expired_keys.push(key.to_vec());
+            }
+            Ok(true)
+        })
+        .await
+        .unwrap();
+
+    if !expired_keys.is_empty() {
+        let mut batch = BatchBuilder::new();
+        for key in expired_keys {
+            batch.any_op(Operation::Value {
+                class: ValueClass::InMemory(InMemoryClass::Key(key)),
+                op: ValueOp::Clear,
+            });
+            if batch.is_large_batch() {
+                store.write(batch.build_all()).await.unwrap();
+                batch = BatchBuilder::new();
+            }
+        }
+        if !batch.is_empty() {
+            store.write(batch.build_all()).await.unwrap();
+        }
+    }
+
+    if !expired_counters.is_empty() {
+        let mut batch = BatchBuilder::new();
+        for key in expired_counters {
+            batch.any_op(Operation::Value {
+                class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())),
+                op: ValueOp::Clear,
+            });
+            batch.any_op(Operation::Value {
+                class: ValueClass::InMemory(InMemoryClass::Key(key)),
+                op: ValueOp::Clear,
+            });
+            if batch.is_large_batch() {
+                store.write(batch.build_all()).await.unwrap();
+                batch = BatchBuilder::new();
+            }
+        }
+        if !batch.is_empty() {
+            store.write(batch.build_all()).await.unwrap();
+        }
+    }
+}
+
+#[allow(unused_variables)]
+pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include_registry: bool) {
+    store_blob_expire_all(store).await;
+    store_lookup_expire_all(store).await;
+    for shard_idx in 0..=u8::MAX {
+        store
+            .purge_blobs(blob_store.clone(), shard_idx)
+            .await
+            .unwrap();
+    }
+    store.purge_store().await.unwrap();
+
+    let store = store.clone();
+    let mut failed = false;
+    let mut delete_batch = BatchBuilder::new();
+
+    for (subspace, with_values) in [
+        (SUBSPACE_ACL, true),
+        (SUBSPACE_TASK_QUEUE, true),
+        (SUBSPACE_IN_MEMORY_VALUE, true),
+        (SUBSPACE_IN_MEMORY_COUNTER, false),
+        (SUBSPACE_PROPERTY, true),
+        (SUBSPACE_QUEUE_MESSAGE, true),
+        (SUBSPACE_QUEUE_EVENT, true),
+        (SUBSPACE_REPORT_OUT, true),
+        (SUBSPACE_REPORT_IN, true),
+        (SUBSPACE_DELETED_ITEMS, true),
+        (SUBSPACE_SPAM_SAMPLES, true),
+        (SUBSPACE_BLOB_LINK, true),
+        (SUBSPACE_BLOBS, true),
+        (SUBSPACE_COUNTER, false),
+        (SUBSPACE_QUOTA, false),
+        (SUBSPACE_INDEXES, false),
+        (SUBSPACE_TELEMETRY_SPAN, true),
+        (SUBSPACE_TELEMETRY_METRIC, true),
+        (SUBSPACE_SEARCH_INDEX, true),
+        (SUBSPACE_REGISTRY, true),
+        (SUBSPACE_REGISTRY_IDX, false),
+        (SUBSPACE_REGISTRY_PK, true),
+        (SUBSPACE_DIRECTORY, true),
+    ] {
+        if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() {
+            continue;
+        }
+
+        let from_key = AnyKey {
+            subspace,
+            key: vec![0u8],
+        };
+        let to_key = AnyKey {
+            subspace,
+            key: vec![u8::MAX; 10],
+        };
+
+        store
+            .iterate(
+                IterateParams::new(from_key, to_key).set_values(with_values),
+                |key, value| {
+                    match subspace {
+                        SUBSPACE_COUNTER
+                            if key.len() == U32_LEN + 1
+                                || key.len() == U32_LEN
+                                || key.len() == U16_LEN =>
+                        {
+                            // Message ID, change ID counters and registry counters
+                            if key.len() != U16_LEN {
+                                // Keep registry counters, delete the rest
+                                delete_batch.clear(ValueClass::Any(AnyClass {
+                                    subspace,
+                                    key: key.to_vec(),
+                                }));
+                            }
+                            return Ok(true);
+                        }
+                        SUBSPACE_INDEXES => {
+                            println!(
+                                concat!(
+                                    "Found index key, account {}, collection {}, ",
+                                    "document {}, property {}, value {:?}: {:?}"
+                                ),
+                                u32::from_be_bytes(key[0..4].try_into().unwrap()),
+                                key[4],
+                                u32::from_be_bytes(key[key.len() - 4..].try_into().unwrap()),
+                                key[5],
+                                String::from_utf8_lossy(&key[6..key.len() - 4]),
+                                key
+                            );
+                        }
+                        SUBSPACE_REGISTRY | SUBSPACE_DIRECTORY | SUBSPACE_SPAM_SAMPLES => {
+                            let object_id =
+                                ObjectType::from_id(key.deserialize_be_u16(0).unwrap()).unwrap();
+
+                            if include_registry && is_allowed_registry_type(object_id) {
+                                return Ok(true);
+                            }
+                            let item_id = key.deserialize_be_u64(U16_LEN).unwrap();
+
+                            println!(
+                                "Found registry item for object type {:?} and id {}",
+                                object_id, item_id
+                            );
+                        }
+                        SUBSPACE_REGISTRY_IDX => {
+                            let mut id = key.deserialize_be_u16(0).unwrap();
+                            if id == u16::MAX {
+                                id = key.deserialize_be_u16(U16_LEN).unwrap();
+                            }
+
+                            let object_id = ObjectType::from_id(id).unwrap();
+
+                            if include_registry && is_allowed_registry_type(object_id) {
+                                return Ok(true);
+                            }
+
+                            println!(
+                                "Found registry index for object type {:?}: {:?}",
+                                object_id, key
+                            );
+                        }
+                        SUBSPACE_REGISTRY_PK => {
+                            let mut id = key.deserialize_be_u16(0).unwrap();
+                            if id == u16::MAX {
+                                id = value.deserialize_be_u16(0).unwrap();
+                            }
+                            let object_id = ObjectType::from_id(id).unwrap();
+                            if include_registry && is_allowed_registry_type(object_id) {
+                                return Ok(true);
+                            }
+
+                            println!(
+                                "Found registry primary key for object type {:?}: {:?}",
+                                object_id, key
+                            );
+                        }
+                        _ => {
+                            println!(
+                                "Found key in {:?}: {:?} ({:?}) = {:?} ({:?})",
+                                char::from(subspace),
+                                key,
+                                String::from_utf8_lossy(key),
+                                value,
+                                String::from_utf8_lossy(value)
+                            );
+                        }
+                    }
+                    failed = true;
+
+                    Ok(true)
+                },
+            )
+            .await
+            .unwrap();
+    }
+
+    // Delete logs and counters
+    store
+        .delete_range(
+            AnyKey {
+                subspace: SUBSPACE_LOGS,
+                key: &[0u8],
+            },
+            AnyKey {
+                subspace: SUBSPACE_LOGS,
+                key: &[
+                    u8::MAX,
+                    u8::MAX,
+                    u8::MAX,
+                    u8::MAX,
+                    u8::MAX,
+                    u8::MAX,
+                    u8::MAX,
+                ],
+            },
+        )
+        .await
+        .unwrap();
+
+    if !delete_batch.is_empty() {
+        store.write(delete_batch.build_all()).await.unwrap();
+    }
+
+    if failed {
+        panic!("Store is not empty.");
+    }
+}
+
+fn is_allowed_registry_type(object_type: ObjectType) -> bool {
+    (object_type.flags() & OBJ_SINGLETON) != 0
+        || matches!(
+            object_type,
+            ObjectType::Role
+                | ObjectType::Account
+                | ObjectType::NetworkListener
+                | ObjectType::MtaDeliverySchedule
+                | ObjectType::MtaRoute
+                | ObjectType::MtaTlsStrategy
+                | ObjectType::MtaVirtualQueue
+                | ObjectType::MtaConnectionStrategy
+                | ObjectType::MtaInboundThrottle
+                | ObjectType::MtaQueueQuota
+                | ObjectType::Tracer
+                | ObjectType::Domain
+        )
+}

tests/src/utils/containers.rs

@@ -0,0 +1,540 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::time::{Duration, Instant};
+
+use testcontainers::{
+    ContainerAsync, GenericBuildableImage, GenericImage, ImageExt, ReuseDirective,
+    core::{CmdWaitFor, ExecCommand, Host, IntoContainerPort, WaitFor},
+    runners::{AsyncBuilder, AsyncRunner},
+};
+use tokio::{net::TcpStream, sync::OnceCell};
+
+const ACME_NETWORK: &str = "stalwart-test-acme";
+
+const READY_TIMEOUT: Duration = Duration::from_secs(180);
+
+static FOUNDATIONDB: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static POSTGRES: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static MYSQL: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static MARIADB: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static REDIS: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static NATS: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static MINIO: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static OPENSEARCH: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static MEILISEARCH: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static KEYCLOAK: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static OPENLDAP: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static CHALLTESTSRV: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static PEBBLE: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static POWERDNS: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static SCIM_TESTER: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+
+const OPENLDAP_LDAPI_URL: &str = "ldapi://%2Fvar%2Frun%2Fslapd%2Fldapi/";
+
+const OPENLDAP_ALLOW_UNAUTHENTICATED_BIND: &str = r#"set -e
+ldapmodify -Y EXTERNAL -Q -H "$LDAPI" <<'EOF'
+dn: cn=config
+changetype: modify
+replace: olcAllows
+olcAllows: bind_anon_dn
+EOF
+test "$(ldapwhoami -x -H "$LDAPI" -D uid=john.doe,ou=users,dc=stalwart,dc=test -w '')" = anonymous
+"#;
+
+const POWERDNS_ZONE_INIT: &str = r#"set -e
+for i in $(seq 1 60); do
+    pdnsutil list-all-zones >/dev/null 2>&1 && break
+    sleep 1
+done
+if pdnsutil list-zone stalwart.test >/dev/null 2>&1; then
+    exit 0
+fi
+pdnsutil create-zone stalwart.test ns1.stalwart.test
+pdnsutil set-kind stalwart.test native
+pdnsutil replace-rrset stalwart.test '' SOA 'ns1.stalwart.test. admin.stalwart.test. 2024010101 3600 900 604800 86400'
+pdnsutil add-record stalwart.test 'ns1' A '127.0.0.1'
+pdnsutil add-record stalwart.test '' A '127.0.0.1'
+pdnsutil add-record stalwart.test '' MX '10 mail.stalwart.test.'
+pdnsutil add-record stalwart.test 'mail' A '127.0.0.1'
+pdnsutil import-tsig-key stalwart-update-key hmac-sha256 'c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldC1rZXkxMjM0NTY3ODkw'
+pdnsutil activate-tsig-key stalwart.test stalwart-update-key primary
+pdnsutil set-meta stalwart.test TSIG-ALLOW-DNSUPDATE stalwart-update-key
+pdnsutil set-meta stalwart.test ALLOW-DNSUPDATE-FROM '0.0.0.0/0'
+"#;
+
+pub async fn ensure_foundationdb() {
+    let container = FOUNDATIONDB
+        .get_or_init(|| async {
+            GenericImage::new("foundationdb/foundationdb", "7.4.6")
+                .with_env_var("FDB_NETWORKING_MODE", "container")
+                .with_mapped_port(4500, 4500.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-foundationdb")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start FoundationDB container")
+        })
+        .await;
+
+    let start = Instant::now();
+    loop {
+        if fdbcli(container, "status minimal")
+            .await
+            .contains("The database is available")
+        {
+            return;
+        }
+        let created = fdbcli(container, "configure new single memory").await;
+        if created.contains("Database created") || created.contains("Already exists") {
+            continue;
+        }
+        if start.elapsed() > READY_TIMEOUT {
+            panic!("Timed out configuring FoundationDB: {created}");
+        }
+        tokio::time::sleep(Duration::from_secs(2)).await;
+    }
+}
+
+async fn fdbcli(container: &ContainerAsync<GenericImage>, command: &str) -> String {
+    let mut result = container
+        .exec(
+            ExecCommand::new(["fdbcli", "--exec", command, "--timeout", "5"])
+                .with_cmd_ready_condition(CmdWaitFor::exit()),
+        )
+        .await
+        .expect("Failed to exec fdbcli");
+    let stdout = result.stdout_to_vec().await.unwrap_or_default();
+    let stderr = result.stderr_to_vec().await.unwrap_or_default();
+    format!(
+        "{}{}",
+        String::from_utf8_lossy(&stdout),
+        String::from_utf8_lossy(&stderr)
+    )
+}
+
+pub async fn ensure_postgres() {
+    POSTGRES
+        .get_or_init(|| async {
+            GenericImage::new("postgres", "16-alpine")
+                .with_wait_for(WaitFor::message_on_stderr(
+                    "database system is ready to accept connections",
+                ))
+                .with_wait_for(WaitFor::message_on_stderr(
+                    "database system is ready to accept connections",
+                ))
+                .with_env_var("POSTGRES_USER", "stalwart")
+                .with_env_var("POSTGRES_PASSWORD", "stalwart")
+                .with_env_var("POSTGRES_DB", "stalwart")
+                .with_mapped_port(5432, 5432.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-postgres")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start PostgreSQL container")
+        })
+        .await;
+    wait_for_tcp(5432).await;
+}
+
+pub async fn ensure_mysql() {
+    MYSQL
+        .get_or_init(|| async {
+            GenericImage::new("mysql", "8.0")
+                .with_wait_for(WaitFor::message_on_stderr("port: 3306  MySQL"))
+                .with_env_var("MYSQL_ROOT_PASSWORD", "password")
+                .with_env_var("MYSQL_DATABASE", "stalwart")
+                .with_cmd(["--default-authentication-plugin=mysql_native_password"])
+                .with_mapped_port(3307, 3306.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-mysql")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start MySQL container")
+        })
+        .await;
+    wait_for_tcp(3307).await;
+}
+
+pub async fn ensure_mariadb() {
+    MARIADB
+        .get_or_init(|| async {
+            GenericImage::new("mariadb", "11.4")
+                .with_wait_for(WaitFor::message_on_stderr("port: 3306  mariadb.org"))
+                .with_env_var("MARIADB_ROOT_PASSWORD", "password")
+                .with_env_var("MARIADB_DATABASE", "stalwart")
+                .with_mapped_port(3308, 3306.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-mariadb")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start MariaDB container")
+        })
+        .await;
+    wait_for_tcp(3308).await;
+}
+
+pub async fn ensure_redis() {
+    REDIS
+        .get_or_init(|| async {
+            GenericImage::new("redis", "7-alpine")
+                .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections"))
+                .with_cmd(["redis-server", "--save", "", "--appendonly", "no"])
+                .with_mapped_port(6379, 6379.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-redis")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start Redis container")
+        })
+        .await;
+    wait_for_tcp(6379).await;
+}
+
+pub async fn ensure_nats() {
+    NATS.get_or_init(|| async {
+        GenericImage::new("nats", "latest")
+            .with_wait_for(WaitFor::message_on_stderr("Server is ready"))
+            .with_cmd(["--addr", "0.0.0.0", "--port", "4222", "--http_port", "8222"])
+            .with_mapped_port(4222, 4222.tcp())
+            .with_mapped_port(8222, 8222.tcp())
+            .with_startup_timeout(READY_TIMEOUT)
+            .with_container_name("stalwart-test-nats")
+            .with_reuse(ReuseDirective::Always)
+            .start()
+            .await
+            .expect("Failed to start NATS container")
+    })
+    .await;
+    wait_for_tcp(4222).await;
+}
+
+pub async fn ensure_minio() {
+    MINIO
+        .get_or_init(|| async {
+            GenericImage::new("minio/minio", "latest")
+                .with_env_var("MINIO_ROOT_USER", "minioadmin")
+                .with_env_var("MINIO_ROOT_PASSWORD", "minioadmin")
+                .with_cmd(["server", "/data", "--console-address", ":9001"])
+                .with_mapped_port(9000, 9000.tcp())
+                .with_mapped_port(9001, 9001.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-minio")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start MinIO container")
+        })
+        .await;
+    wait_for_http("http://localhost:9000/minio/health/live").await;
+    create_minio_bucket().await;
+}
+
+pub async fn ensure_opensearch() {
+    OPENSEARCH
+        .get_or_init(|| async {
+            GenericImage::new("opensearchproject/opensearch", "2")
+                .with_env_var("discovery.type", "single-node")
+                .with_env_var("DISABLE_SECURITY_PLUGIN", "true")
+                .with_env_var("OPENSEARCH_JAVA_OPTS", "-Xms1g -Xmx1g")
+                .with_env_var("DISABLE_INSTALL_DEMO_CONFIG", "true")
+                .with_mapped_port(9200, 9200.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-opensearch")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start OpenSearch container")
+        })
+        .await;
+    wait_for_http("http://localhost:9200").await;
+}
+
+pub async fn ensure_meilisearch() {
+    MEILISEARCH
+        .get_or_init(|| async {
+            GenericImage::new("getmeili/meilisearch", "latest")
+                .with_env_var("MEILI_ENV", "development")
+                .with_env_var("MEILI_NO_ANALYTICS", "true")
+                .with_env_var("MEILI_MASTER_KEY", "stalwart-master-key")
+                .with_mapped_port(7700, 7700.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-meilisearch")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start Meilisearch container")
+        })
+        .await;
+    wait_for_http("http://localhost:7700/health").await;
+}
+
+pub async fn ensure_keycloak() {
+    KEYCLOAK
+        .get_or_init(|| async {
+            GenericImage::new("quay.io/keycloak/keycloak", "latest")
+                .with_env_var("KC_BOOTSTRAP_ADMIN_USERNAME", "admin")
+                .with_env_var("KC_BOOTSTRAP_ADMIN_PASSWORD", "admin")
+                .with_env_var("KC_HTTP_PORT", "9080")
+                .with_env_var("KC_HEALTH_ENABLED", "true")
+                .with_cmd(["start-dev", "--import-realm"])
+                .with_copy_to(
+                    "/opt/keycloak/data/import/stalwart-realm.json",
+                    include_bytes!("../../docker/keycloak/stalwart-realm.json").to_vec(),
+                )
+                .with_mapped_port(9080, 9080.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-keycloak")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start Keycloak container")
+        })
+        .await;
+    wait_for_http("http://localhost:9080/realms/stalwart/.well-known/openid-configuration").await;
+}
+
+pub async fn ensure_scim_tester() -> &'static ContainerAsync<GenericImage> {
+    SCIM_TESTER
+        .get_or_init(|| async {
+            let image = GenericBuildableImage::new("stalwart-test-scim-tester", "local")
+                .with_dockerfile_string(include_str!("../../docker/scim/Dockerfile"))
+                .build_image()
+                .await
+                .expect("Failed to build the SCIM tester image");
+            image
+                .with_host("host.docker.internal", Host::HostGateway)
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-scim-tester")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start the SCIM tester container")
+        })
+        .await
+}
+
+pub async fn scim_tester_exec(args: &[&str]) -> (String, String) {
+    let mut result = ensure_scim_tester()
+        .await
+        .exec(ExecCommand::new(args.iter().copied()).with_cmd_ready_condition(CmdWaitFor::exit()))
+        .await
+        .expect("Failed to exec the SCIM driver");
+    let stdout = result.stdout_to_vec().await.unwrap_or_default();
+    let stderr = result.stderr_to_vec().await.unwrap_or_default();
+
+    (
+        String::from_utf8_lossy(&stdout).into_owned(),
+        String::from_utf8_lossy(&stderr).into_owned(),
+    )
+}
+
+pub async fn ensure_acme() {
+    ensure_challtestsrv().await;
+    ensure_pebble().await;
+}
+
+async fn ensure_challtestsrv() {
+    CHALLTESTSRV
+        .get_or_init(|| async {
+            let image = GenericBuildableImage::new("stalwart-test-challtestsrv", "local")
+                .with_dockerfile_string(include_str!("../../docker/pebble/Dockerfile.challtestsrv"))
+                .build_image()
+                .await
+                .expect("Failed to build challtestsrv image");
+            image
+                .with_network(ACME_NETWORK)
+                .with_host("host.docker.internal", Host::HostGateway)
+                .with_mapped_port(8055, 8055.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-challtestsrv")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start challtestsrv container")
+        })
+        .await;
+    wait_for_tcp(8055).await;
+}
+
+async fn ensure_pebble() {
+    PEBBLE
+        .get_or_init(|| async {
+            GenericImage::new("ghcr.io/letsencrypt/pebble", "latest")
+                .with_env_var("PEBBLE_VA_NOSLEEP", "1")
+                .with_env_var("PEBBLE_WFE_NONCEREJECT", "0")
+                .with_env_var("PEBBLE_ALTERNATE_ROOTS", "2")
+                .with_cmd([
+                    "-config",
+                    "/test/config/pebble-config.json",
+                    "-dnsserver",
+                    "stalwart-test-challtestsrv:8053",
+                ])
+                .with_copy_to(
+                    "/test/config/pebble-config.json",
+                    include_bytes!("../../docker/pebble/pebble-config.json").to_vec(),
+                )
+                .with_network(ACME_NETWORK)
+                .with_host("host.docker.internal", Host::HostGateway)
+                .with_mapped_port(14000, 14000.tcp())
+                .with_mapped_port(15000, 15000.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-pebble")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start Pebble container")
+        })
+        .await;
+    wait_for_tcp(14000).await;
+}
+
+pub async fn ensure_powerdns() {
+    let container = POWERDNS
+        .get_or_init(|| async {
+            GenericImage::new("powerdns/pdns-auth-49", "latest")
+                .with_wait_for(WaitFor::message_on_stderr("Creating backend connection"))
+                .with_env_var("PDNS_AUTH_API_KEY", "stalwart-api-key")
+                .with_copy_to(
+                    "/etc/powerdns/pdns.d/stalwart.conf",
+                    include_bytes!("../../docker/powerdns/pdns.conf").to_vec(),
+                )
+                .with_mapped_port(5300, 53.tcp())
+                .with_mapped_port(5300, 53.udp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-powerdns")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start PowerDNS container")
+        })
+        .await;
+
+    let mut result = container
+        .exec(
+            ExecCommand::new(["bash", "-c", POWERDNS_ZONE_INIT])
+                .with_cmd_ready_condition(CmdWaitFor::exit()),
+        )
+        .await
+        .expect("Failed to exec PowerDNS zone init");
+    if result.exit_code().await.ok().flatten() != Some(0) {
+        let stdout =
+            String::from_utf8_lossy(&result.stdout_to_vec().await.unwrap_or_default()).into_owned();
+        let stderr =
+            String::from_utf8_lossy(&result.stderr_to_vec().await.unwrap_or_default()).into_owned();
+        panic!("PowerDNS zone init failed:\n{stdout}\n{stderr}");
+    }
+    wait_for_tcp(5300).await;
+}
+
+pub async fn ensure_openldap() {
+    const BOOTSTRAP_DIR: &str = "/container/service/slapd/assets/config/bootstrap/ldif/custom";
+    let container = OPENLDAP
+        .get_or_init(|| async {
+            GenericImage::new("osixia/openldap", "1.5.0")
+                .with_wait_for(WaitFor::message_on_stderr("slapd starting"))
+                .with_env_var("LDAP_ORGANISATION", "Stalwart Test")
+                .with_env_var("LDAP_DOMAIN", "stalwart.test")
+                .with_env_var("LDAP_BASE_DN", "dc=stalwart,dc=test")
+                .with_env_var("LDAP_ADMIN_PASSWORD", "admin")
+                .with_env_var("LDAP_TLS", "false")
+                .with_copy_to(
+                    format!("{BOOTSTRAP_DIR}/50-users.ldif"),
+                    include_bytes!("../../docker/ldap/50-users.ldif").to_vec(),
+                )
+                .with_copy_to(
+                    format!("{BOOTSTRAP_DIR}/60-groups.ldif"),
+                    include_bytes!("../../docker/ldap/60-groups.ldif").to_vec(),
+                )
+                .with_mapped_port(389, 389.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name("stalwart-test-openldap")
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start OpenLDAP container")
+        })
+        .await;
+    wait_for_tcp(389).await;
+
+    let setup = format!("LDAPI={OPENLDAP_LDAPI_URL}\n{OPENLDAP_ALLOW_UNAUTHENTICATED_BIND}");
+    let mut result = container
+        .exec(
+            ExecCommand::new(["bash", "-c", setup.as_str()])
+                .with_cmd_ready_condition(CmdWaitFor::exit()),
+        )
+        .await
+        .expect("Failed to exec OpenLDAP unauthenticated bind setup");
+    if result.exit_code().await.ok().flatten() != Some(0) {
+        let stdout =
+            String::from_utf8_lossy(&result.stdout_to_vec().await.unwrap_or_default()).into_owned();
+        let stderr =
+            String::from_utf8_lossy(&result.stderr_to_vec().await.unwrap_or_default()).into_owned();
+        panic!("OpenLDAP unauthenticated bind setup failed:\n{stdout}\n{stderr}");
+    }
+}
+
+async fn create_minio_bucket() {
+    use s3::{Bucket, BucketConfiguration, Region, creds::Credentials};
+
+    let region = Region::Custom {
+        region: "eu-central-1".to_string(),
+        endpoint: "http://localhost:9000".to_string(),
+    };
+    let credentials = Credentials::new(Some("minioadmin"), Some("minioadmin"), None, None, None)
+        .expect("Failed to build MinIO credentials");
+
+    match Bucket::create_with_path_style(
+        "stalwart",
+        region,
+        credentials,
+        BucketConfiguration::default(),
+    )
+    .await
+    {
+        Ok(response) if response.success() => {}
+        Ok(_) => {}
+        Err(s3::error::S3Error::HttpFailWithBody(409, _)) => {}
+        Err(err) => panic!("Failed to create MinIO bucket: {err:?}"),
+    }
+}
+
+async fn wait_for_tcp(port: u16) {
+    let start = Instant::now();
+    loop {
+        if TcpStream::connect(("127.0.0.1", port)).await.is_ok() {
+            return;
+        }
+        if start.elapsed() > READY_TIMEOUT {
+            panic!("Timed out waiting for TCP port {port}");
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+}
+
+async fn wait_for_http(url: &str) {
+    let client = reqwest::Client::builder()
+        .danger_accept_invalid_certs(true)
+        .build()
+        .expect("Failed to build HTTP client");
+    let start = Instant::now();
+    loop {
+        if let Ok(response) = client.get(url).send().await
+            && response.status().is_success()
+        {
+            return;
+        }
+        if start.elapsed() > READY_TIMEOUT {
+            panic!("Timed out waiting for {url}");
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+}

tests/src/utils/dns.rs

@@ -0,0 +1,148 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::{
+    Server,
+    config::{mailstore::spamfilter::IpResolver, smtp::resolver::Tlsa},
+};
+use mail_auth::{DnssecStatus, MX, RecordSet, Txt, common::resolver::ToFqdn};
+use std::{
+    net::{IpAddr, Ipv4Addr, Ipv6Addr},
+    sync::Arc,
+};
+
+pub trait DnsCache {
+    fn txt_add(&self, name: impl ToFqdn, value: impl Into<Txt>, valid_until: std::time::Instant);
+    fn ipv4_add(&self, name: impl ToFqdn, value: Vec<Ipv4Addr>, valid_until: std::time::Instant);
+    fn ipv6_add(&self, name: impl ToFqdn, value: Vec<Ipv6Addr>, valid_until: std::time::Instant);
+    fn ipv4_add_dnssec(
+        &self,
+        name: impl ToFqdn,
+        value: Vec<Ipv4Addr>,
+        dnssec_status: DnssecStatus,
+        valid_until: std::time::Instant,
+    );
+    fn ipv6_add_dnssec(
+        &self,
+        name: impl ToFqdn,
+        value: Vec<Ipv6Addr>,
+        dnssec_status: DnssecStatus,
+        valid_until: std::time::Instant,
+    );
+    fn dnsbl_add(&self, name: &str, value: Vec<Ipv4Addr>, valid_until: std::time::Instant);
+    fn ptr_add(&self, name: IpAddr, value: Vec<String>, valid_until: std::time::Instant);
+    fn mx_add(
+        &self,
+        name: impl ToFqdn,
+        value: Vec<MX>,
+        dnssec_status: DnssecStatus,
+        valid_until: std::time::Instant,
+    );
+    fn tlsa_add(&self, name: impl ToFqdn, value: Arc<Tlsa>, valid_until: std::time::Instant);
+}
+
+impl DnsCache for Server {
+    fn txt_add(&self, name: impl ToFqdn, value: impl Into<Txt>, valid_until: std::time::Instant) {
+        self.inner.cache.dns_txt.insert_with_expiry(
+            name.to_fqdn().into_owned().into_boxed_str(),
+            value.into(),
+            valid_until,
+        );
+    }
+
+    fn ipv4_add(&self, name: impl ToFqdn, value: Vec<Ipv4Addr>, valid_until: std::time::Instant) {
+        self.ipv4_add_dnssec(name, value, DnssecStatus::Secure, valid_until);
+    }
+
+    fn ipv4_add_dnssec(
+        &self,
+        name: impl ToFqdn,
+        value: Vec<Ipv4Addr>,
+        dnssec_status: DnssecStatus,
+        valid_until: std::time::Instant,
+    ) {
+        self.inner.cache.dns_ipv4.insert_with_expiry(
+            name.to_fqdn().into_owned().into_boxed_str(),
+            RecordSet {
+                rrset: Arc::from(value),
+                dnssec_status,
+            },
+            valid_until,
+        );
+    }
+
+    fn dnsbl_add(&self, name: &str, value: Vec<Ipv4Addr>, valid_until: std::time::Instant) {
+        self.inner.cache.dns_rbl.insert_with_expiry(
+            name.into(),
+            Some(Arc::new(IpResolver::new(
+                value
+                    .iter()
+                    .copied()
+                    .next()
+                    .unwrap_or(Ipv4Addr::BROADCAST)
+                    .into(),
+            ))),
+            valid_until,
+        );
+    }
+
+    fn ipv6_add(&self, name: impl ToFqdn, value: Vec<Ipv6Addr>, valid_until: std::time::Instant) {
+        self.ipv6_add_dnssec(name, value, DnssecStatus::Secure, valid_until);
+    }
+
+    fn ipv6_add_dnssec(
+        &self,
+        name: impl ToFqdn,
+        value: Vec<Ipv6Addr>,
+        dnssec_status: DnssecStatus,
+        valid_until: std::time::Instant,
+    ) {
+        self.inner.cache.dns_ipv6.insert_with_expiry(
+            name.to_fqdn().into_owned().into_boxed_str(),
+            RecordSet {
+                rrset: Arc::from(value),
+                dnssec_status,
+            },
+            valid_until,
+        );
+    }
+
+    fn ptr_add(&self, name: IpAddr, value: Vec<String>, valid_until: std::time::Instant) {
+        self.inner.cache.dns_ptr.insert_with_expiry(
+            name,
+            RecordSet {
+                rrset: Arc::from(value.into_iter().map(Into::into).collect::<Vec<_>>()),
+                dnssec_status: DnssecStatus::Indeterminate,
+            },
+            valid_until,
+        );
+    }
+
+    fn mx_add(
+        &self,
+        name: impl ToFqdn,
+        value: Vec<MX>,
+        dnssec_status: DnssecStatus,
+        valid_until: std::time::Instant,
+    ) {
+        self.inner.cache.dns_mx.insert_with_expiry(
+            name.to_fqdn().into_owned().into_boxed_str(),
+            RecordSet {
+                rrset: Arc::from(value),
+                dnssec_status,
+            },
+            valid_until,
+        );
+    }
+
+    fn tlsa_add(&self, name: impl ToFqdn, value: Arc<Tlsa>, valid_until: std::time::Instant) {
+        self.inner.cache.dns_tlsa.insert_with_expiry(
+            name.to_fqdn().into_owned().into_boxed_str(),
+            value,
+            valid_until,
+        );
+    }
+}

tests/src/utils/http.rs

@@ -0,0 +1,189 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use hyper::Method;
+use reqwest::header::HeaderMap;
+use serde::{Serialize, de::DeserializeOwned};
+use std::time::Duration;
+
+pub struct HttpRequest {
+    pub port: u16,
+    pub username: Option<String>,
+    pub password: Option<String>,
+}
+
+pub struct HttpResponseFull {
+    pub status: reqwest::StatusCode,
+    pub headers: HeaderMap,
+    pub body: String,
+}
+
+impl HttpResponseFull {
+    pub fn header(&self, name: &str) -> Option<&str> {
+        self.headers.get(name).and_then(|v| v.to_str().ok())
+    }
+
+    pub fn rate_limit_policy(&self) -> Option<&str> {
+        self.header("RateLimit-Policy")
+    }
+
+    pub fn rate_limit(&self) -> Option<&str> {
+        self.header("RateLimit")
+    }
+
+    pub fn retry_after(&self) -> Option<u64> {
+        self.header("Retry-After").and_then(|v| v.parse().ok())
+    }
+}
+
+impl Default for HttpRequest {
+    fn default() -> Self {
+        Self {
+            port: 8899,
+            username: None,
+            password: None,
+        }
+    }
+}
+
+impl HttpRequest {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn with_credentials(port: u16, username: &str, password: &str) -> Self {
+        Self {
+            port,
+            username: Some(username.to_string()),
+            password: Some(password.to_string()),
+        }
+    }
+
+    pub async fn post<T: DeserializeOwned>(
+        &self,
+        query: &str,
+        body: &impl Serialize,
+    ) -> Result<T, String> {
+        self.request_raw(
+            Method::POST,
+            query,
+            Some(serde_json::to_string(body).unwrap()),
+        )
+        .await
+        .map(|result| {
+            serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+        })
+    }
+
+    pub async fn patch<T: DeserializeOwned>(
+        &self,
+        query: &str,
+        body: &impl Serialize,
+    ) -> Result<T, String> {
+        self.request_raw(
+            Method::PATCH,
+            query,
+            Some(serde_json::to_string(body).unwrap()),
+        )
+        .await
+        .map(|result| {
+            serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+        })
+    }
+
+    pub async fn delete<T: DeserializeOwned>(&self, query: &str) -> Result<T, String> {
+        self.request_raw(Method::DELETE, query, None)
+            .await
+            .map(|result| {
+                serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+            })
+    }
+
+    pub async fn get<T: DeserializeOwned>(&self, query: &str) -> Result<T, String> {
+        self.request_raw(Method::GET, query, None)
+            .await
+            .map(|result| {
+                serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+            })
+    }
+    pub async fn request<T: DeserializeOwned>(
+        &self,
+        method: Method,
+        query: &str,
+    ) -> Result<T, String> {
+        self.request_raw(method, query, None).await.map(|result| {
+            serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+        })
+    }
+
+    pub async fn send_full(
+        &self,
+        method: Method,
+        query: &str,
+        body: Option<Vec<u8>>,
+        content_type: Option<&str>,
+    ) -> HttpResponseFull {
+        let mut request = reqwest::Client::builder()
+            .timeout(Duration::from_secs(5))
+            .danger_accept_invalid_certs(true)
+            .build()
+            .unwrap()
+            .request(method, format!("https://127.0.0.1:{}{query}", self.port));
+
+        if let Some(body) = body {
+            request = request.body(body);
+        }
+
+        if let Some(ct) = content_type {
+            request = request.header(hyper::header::CONTENT_TYPE, ct);
+        }
+
+        if let (Some(username), Some(password)) = (&self.username, &self.password) {
+            request = request.basic_auth(username, Some(password));
+        }
+
+        let response = request.send().await.expect("HTTP request failed");
+        let status = response.status();
+        let headers = response.headers().clone();
+        let body = response.text().await.unwrap_or_default();
+        HttpResponseFull {
+            status,
+            headers,
+            body,
+        }
+    }
+
+    async fn request_raw(
+        &self,
+        method: Method,
+        query: &str,
+        body: Option<String>,
+    ) -> Result<String, String> {
+        let mut request = reqwest::Client::builder()
+            .timeout(Duration::from_millis(500))
+            .danger_accept_invalid_certs(true)
+            .build()
+            .unwrap()
+            .request(method, format!("https://127.0.0.1:{}{query}", self.port));
+
+        if let Some(body) = body {
+            request = request.body(body);
+        }
+
+        if let (Some(username), Some(password)) = (&self.username, &self.password) {
+            request = request.basic_auth(username, Some(password));
+        }
+
+        request
+            .send()
+            .await
+            .map_err(|err| err.to_string())?
+            .bytes()
+            .await
+            .map(|bytes| String::from_utf8(bytes.to_vec()).unwrap())
+            .map_err(|err| err.to_string())
+    }
+}

tests/src/utils/http_server.rs

@@ -0,0 +1,134 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{AssertConfig, utils::server::TestServer};
+use ahash::AHashMap;
+use common::{config::server::Listeners, network::SessionData};
+use http_proto::{HttpResponse, request::fetch_body};
+use hyper::{Method, Uri, body, server::conn::http1, service::service_fn};
+use hyper_util::rt::TokioIo;
+use registry::{
+    schema::{
+        enums::NetworkListenerProtocol,
+        prelude::{ObjectType, SocketAddr},
+        structs::{NetworkListener, SystemSettings},
+    },
+    types::{id::ObjectId, map::Map},
+};
+use std::{str::FromStr, sync::Arc};
+use store::registry::{RegistryObject, bootstrap::Bootstrap};
+use tokio::sync::watch;
+
+#[derive(Clone)]
+pub struct HttpSessionManager {
+    inner: HttpRequestHandler,
+}
+
+pub type HttpRequestHandler = Arc<dyn Fn(HttpMessage) -> HttpResponse + Sync + Send>;
+
+#[derive(Debug)]
+pub struct HttpMessage {
+    pub method: Method,
+    pub headers: AHashMap<String, String>,
+    pub uri: Uri,
+    pub body: Option<Vec<u8>>,
+}
+
+impl HttpMessage {
+    pub fn get_url_encoded(&self, key: &str) -> Option<String> {
+        form_urlencoded::parse(self.body.as_ref()?.as_slice())
+            .find(|(k, _)| k == key)
+            .map(|(_, v)| v.into_owned())
+    }
+}
+
+pub async fn spawn_mock_http_server(
+    test: &TestServer,
+    handler: HttpRequestHandler,
+    port: u16,
+) -> (watch::Sender<bool>, watch::Receiver<bool>) {
+    // Start mock HTTP server
+    let mut bp = Bootstrap::new_uninitialized(test.server.registry().clone());
+    let mut servers = Listeners::default();
+    servers.parse_server(
+        &mut bp,
+        RegistryObject {
+            id: ObjectId::new(ObjectType::NetworkListener, 0u64.into()),
+            object: NetworkListener {
+                name: "mock-http".into(),
+                bind: Map::new(vec![
+                    SocketAddr::from_str(&format!("127.0.0.1:{port}")).unwrap(),
+                ]),
+                protocol: NetworkListenerProtocol::Http,
+                tls_implicit: true,
+                use_tls: true,
+                socket_reuse_address: true,
+                socket_reuse_port: true,
+                ..Default::default()
+            },
+            revision: 0,
+        },
+        &SystemSettings::default(),
+    );
+    servers
+        .parse_tcp_acceptors(&mut bp, test.server.inner.clone())
+        .await;
+    servers.bind_and_drop_priv(&mut bp);
+    bp.assert_no_errors();
+    servers.spawn(|server, acceptor, shutdown_rx| {
+        server.spawn(
+            HttpSessionManager {
+                inner: handler.clone(),
+            },
+            test.server.inner.clone(),
+            acceptor,
+            shutdown_rx,
+        );
+    })
+}
+
+impl common::network::SessionManager for HttpSessionManager {
+    #[allow(clippy::manual_async_fn)]
+    fn handle<T: common::network::SessionStream>(
+        self,
+        session: SessionData<T>,
+    ) -> impl std::future::Future<Output = ()> + Send {
+        async move {
+            let sender = self.inner;
+            let _ = http1::Builder::new()
+                .keep_alive(false)
+                .serve_connection(
+                    TokioIo::new(session.stream),
+                    service_fn(|mut req: hyper::Request<body::Incoming>| {
+                        let sender = sender.clone();
+
+                        async move {
+                            let response = sender(HttpMessage {
+                                method: req.method().clone(),
+                                uri: req.uri().clone(),
+                                headers: req
+                                    .headers()
+                                    .iter()
+                                    .map(|(k, v)| {
+                                        (k.as_str().to_lowercase(), v.to_str().unwrap().to_string())
+                                    })
+                                    .collect(),
+                                body: fetch_body(&mut req, 1024 * 1024, 0).await,
+                            });
+
+                            Ok::<_, hyper::Error>(response.build())
+                        }
+                    }),
+                )
+                .await;
+        }
+    }
+
+    #[allow(clippy::manual_async_fn)]
+    fn shutdown(&self) -> impl std::future::Future<Output = ()> + Send {
+        async {}
+    }
+}

tests/src/utils/imap.rs

@@ -0,0 +1,386 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use base64::{Engine, engine::general_purpose};
+use imap_proto::ResponseType;
+use std::time::Duration;
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+
+pub struct ImapConnection {
+    tag: &'static [u8],
+    reader: BufReader<ReadHalf<TcpStream>>,
+    writer: WriteHalf<TcpStream>,
+    last_raw: Vec<u8>,
+}
+
+async fn read_lossy_line(
+    reader: &mut BufReader<ReadHalf<TcpStream>>,
+) -> std::io::Result<Option<(String, Vec<u8>)>> {
+    let mut buf = Vec::new();
+    let n = reader.read_until(b'\n', &mut buf).await?;
+    if n == 0 {
+        return Ok(None);
+    }
+    let mut trimmed = buf.as_slice();
+    if trimmed.last() == Some(&b'\n') {
+        trimmed = &trimmed[..trimmed.len() - 1];
+    }
+    if trimmed.last() == Some(&b'\r') {
+        trimmed = &trimmed[..trimmed.len() - 1];
+    }
+    Ok(Some((String::from_utf8_lossy(trimmed).into_owned(), buf)))
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Type {
+    Tagged,
+    Untagged,
+    Continuation,
+    Status,
+}
+
+impl ImapConnection {
+    pub async fn connect(tag: &'static [u8]) -> Self {
+        Self::connect_to(tag, "127.0.0.1:9991").await
+    }
+
+    pub async fn connect_to(tag: &'static [u8], addr: impl AsRef<str>) -> Self {
+        let (reader, writer) = tokio::io::split(TcpStream::connect(addr.as_ref()).await.unwrap());
+        ImapConnection {
+            tag,
+            reader: BufReader::new(reader),
+            writer,
+            last_raw: Vec::new(),
+        }
+    }
+
+    pub fn assert_last_contains_bytes(&self, pattern: &[u8]) -> &Self {
+        if !self.last_raw.windows(pattern.len()).any(|w| w == pattern) {
+            panic!(
+                "Expected byte sequence {:02x?} not found in last response ({} bytes).",
+                pattern,
+                self.last_raw.len()
+            );
+        }
+        self
+    }
+
+    pub async fn assert_read(&mut self, t: Type, rt: ResponseType) -> Vec<String> {
+        let lines = self.read(t).await;
+        let mut buf = Vec::with_capacity(10);
+        buf.extend_from_slice(match t {
+            Type::Tagged => self.tag,
+            Type::Untagged | Type::Status => b"* ",
+            Type::Continuation => b"+ ",
+        });
+        if !matches!(t, Type::Continuation | Type::Status) {
+            rt.serialize(&mut buf);
+        }
+        if lines
+            .last()
+            .unwrap()
+            .starts_with(&String::from_utf8(buf).unwrap())
+        {
+            lines
+        } else {
+            panic!("Expected {:?}/{:?} from server but got: {:?}", t, rt, lines);
+        }
+    }
+
+    pub async fn assert_disconnect(&mut self) {
+        match tokio::time::timeout(
+            Duration::from_millis(1500),
+            read_lossy_line(&mut self.reader),
+        )
+        .await
+        {
+            Ok(Ok(None)) => {}
+            Ok(Ok(Some((line, _)))) => {
+                panic!("Expected connection to be closed, but got {:?}", line);
+            }
+            Ok(Err(err)) => {
+                panic!("Connection broken: {:?}", err);
+            }
+            Err(_) => panic!("Timeout while waiting for server response."),
+        }
+    }
+
+    pub async fn read(&mut self, t: Type) -> Vec<String> {
+        let mut lines = Vec::new();
+        self.last_raw.clear();
+        loop {
+            match tokio::time::timeout(
+                Duration::from_millis(1500),
+                read_lossy_line(&mut self.reader),
+            )
+            .await
+            {
+                Ok(Ok(Some((line, raw)))) => {
+                    self.last_raw.extend_from_slice(&raw);
+                    let is_done = line.starts_with(match t {
+                        Type::Tagged => std::str::from_utf8(self.tag).unwrap(),
+                        Type::Untagged | Type::Status => "* ",
+                        Type::Continuation => "+ ",
+                    });
+                    //let c = println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn authenticate(&mut self, user: &str, pass: &str) {
+        let creds = general_purpose::STANDARD.encode(format!("\0{user}\0{pass}"));
+        self.send(&format!(
+            "AUTHENTICATE PLAIN {{{}+}}\r\n{creds}",
+            creds.len()
+        ))
+        .await;
+        self.assert_read(Type::Tagged, ResponseType::Ok).await;
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //let c = println!("-> {}{:?}", std::str::from_utf8(self.tag).unwrap(), text);
+        self.writer.write_all(self.tag).await.unwrap();
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_untagged(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+
+    pub async fn append(&mut self, mailbox: &str, message: &str) {
+        self.send_ok(&format!(
+            "APPEND {:?} {{{}+}}\r\n{}",
+            mailbox,
+            message.len(),
+            message
+        ))
+        .await;
+    }
+
+    pub async fn send_ok(&mut self, cmd: &str) {
+        self.send(cmd).await;
+        self.assert_read(Type::Tagged, ResponseType::Ok).await;
+    }
+}
+
+pub trait AssertResult: Sized {
+    fn assert_folders<'x>(
+        self,
+        expected: impl IntoIterator<Item = (&'x str, impl IntoIterator<Item = &'x str>)>,
+        match_all: bool,
+    ) -> Self;
+
+    fn assert_response_code(self, code: &str) -> Self;
+    fn assert_contains(self, text: &str) -> Self;
+    fn assert_contains_any(self, expected_texts: &[&str]) -> Self;
+    fn assert_not_contains(self, expected_text: &str) -> Self;
+    fn assert_count(self, text: &str, occurrences: usize) -> Self;
+    fn assert_equals(self, text: &str) -> Self;
+    fn into_response_code(self) -> String;
+    fn into_highest_modseq(self) -> String;
+    fn into_uid_validity(self) -> String;
+    fn into_append_uid(self) -> String;
+    fn into_copy_uid(self) -> String;
+    fn into_modseq(self) -> String;
+}
+
+impl AssertResult for Vec<String> {
+    fn assert_folders<'x>(
+        self,
+        expected: impl IntoIterator<Item = (&'x str, impl IntoIterator<Item = &'x str>)>,
+        match_all: bool,
+    ) -> Self {
+        let mut match_count = 0;
+        'outer: for (mailbox_name, flags) in expected.into_iter() {
+            for result in self.iter() {
+                if result.contains(&format!("\"{}\"", mailbox_name)) {
+                    for flag in flags {
+                        if !flag.is_empty() && !result.contains(flag) {
+                            panic!("Expected mailbox {} to have flag {}", mailbox_name, flag);
+                        }
+                    }
+                    match_count += 1;
+                    continue 'outer;
+                }
+            }
+            panic!("Mailbox {} is not present.", mailbox_name);
+        }
+        if match_all && match_count != self.len() - 1 {
+            panic!(
+                "Expected {} mailboxes, but got {}: {:?}",
+                match_count,
+                self.len() - 1,
+                self.iter().collect::<Vec<_>>()
+            );
+        }
+        self
+    }
+
+    fn assert_response_code(self, code: &str) -> Self {
+        if !self.last().unwrap().contains(&format!("[{}]", code)) {
+            panic!(
+                "Response code {:?} not found, got {:?}",
+                code,
+                self.last().unwrap()
+            );
+        }
+        self
+    }
+
+    fn assert_contains(self, expected_text: &str) -> Self {
+        if self.iter().any(|line| line.contains(expected_text)) {
+            self
+        } else {
+            panic!("Expected {:?} but got {}.", expected_text, self.join("\n"));
+        }
+    }
+
+    fn assert_contains_any(self, expected_texts: &[&str]) -> Self {
+        if self
+            .iter()
+            .any(|line| expected_texts.iter().any(|text| line.contains(text)))
+        {
+            self
+        } else {
+            panic!(
+                "Expected any of {:?} but got {}.",
+                expected_texts,
+                self.join("\n")
+            );
+        }
+    }
+
+    fn assert_not_contains(self, expected_text: &str) -> Self {
+        if !self.iter().any(|line| line.contains(expected_text)) {
+            self
+        } else {
+            panic!(
+                "Not expecting {:?} but got it {}.",
+                expected_text,
+                self.join("\n")
+            );
+        }
+    }
+
+    fn assert_count(self, text: &str, occurrences: usize) -> Self {
+        assert_eq!(
+            self.iter().filter(|l| l.contains(text)).count(),
+            occurrences,
+            "Expected {} occurrences of {:?}, found {} in {:?}.",
+            occurrences,
+            text,
+            self.iter().filter(|l| l.contains(text)).count(),
+            self
+        );
+        self
+    }
+
+    fn assert_equals(self, text: &str) -> Self {
+        for line in &self {
+            if line == text {
+                return self;
+            }
+        }
+        panic!("Expected response to be {:?}, got {:?}", text, self);
+    }
+
+    fn into_response_code(self) -> String {
+        if let Some((_, code)) = self.last().unwrap().split_once('[')
+            && let Some((code, _)) = code.split_once(']')
+        {
+            return code.to_string();
+        }
+        panic!("No response code found in {:?}", self.last().unwrap());
+    }
+
+    fn into_append_uid(self) -> String {
+        if let Some((_, code)) = self.last().unwrap().split_once("[APPENDUID ")
+            && let Some((code, _)) = code.split_once(']')
+            && let Some((_, uid)) = code.split_once(' ')
+        {
+            return uid.to_string();
+        }
+        panic!("No APPENDUID found in {:?}", self.last().unwrap());
+    }
+
+    fn into_copy_uid(self) -> String {
+        for line in &self {
+            if let Some((_, code)) = line.split_once("[COPYUID ")
+                && let Some((code, _)) = code.split_once(']')
+                && let Some((_, uid)) = code.rsplit_once(' ')
+            {
+                return uid.to_string();
+            }
+        }
+        panic!("No COPYUID found in {:?}", self);
+    }
+
+    fn into_highest_modseq(self) -> String {
+        for line in &self {
+            if let Some((_, value)) = line.split_once("HIGHESTMODSEQ ") {
+                if let Some((value, _)) = value.split_once(']') {
+                    return value.to_string();
+                } else if let Some((value, _)) = value.split_once(')') {
+                    return value.to_string();
+                } else {
+                    panic!("No HIGHESTMODSEQ delimiter found in {:?}", line);
+                }
+            }
+        }
+        panic!("No HIGHESTMODSEQ entries found in {:?}", self);
+    }
+
+    fn into_modseq(self) -> String {
+        for line in &self {
+            if let Some((_, value)) = line.split_once("MODSEQ (") {
+                if let Some((value, _)) = value.split_once(')') {
+                    return value.to_string();
+                } else {
+                    panic!("No MODSEQ delimiter found in {:?}", line);
+                }
+            }
+        }
+        panic!("No MODSEQ entries found in {:?}", self);
+    }
+
+    fn into_uid_validity(self) -> String {
+        for line in &self {
+            if let Some((_, value)) = line.split_once("UIDVALIDITY ") {
+                if let Some((value, _)) = value.split_once(']') {
+                    return value.to_string();
+                } else if let Some((value, _)) = value.split_once(')') {
+                    return value.to_string();
+                } else {
+                    panic!("No UIDVALIDITY delimiter found in {:?}", line);
+                }
+            }
+        }
+        panic!("No UIDVALIDITY entries found in {:?}", self);
+    }
+}

tests/src/utils/jmap.rs

@@ -0,0 +1,945 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::utils::account::Account;
+use base64::{Engine, engine::general_purpose};
+use hyper::header;
+use jmap_proto::error::set::SetErrorType;
+use registry::types::error::ValidationError;
+use registry::types::id::ObjectId;
+use serde_json::{Value, json};
+use std::{fmt::Display, str::FromStr, time::Duration};
+use types::id::Id;
+
+pub struct JmapResponse(pub Value);
+
+pub struct RawResponse {
+    pub status: u16,
+    pub headers: reqwest::header::HeaderMap,
+    pub body: Vec<u8>,
+}
+
+impl RawResponse {
+    async fn from_response(response: reqwest::Response) -> Self {
+        RawResponse {
+            status: response.status().as_u16(),
+            headers: response.headers().clone(),
+            body: response.bytes().await.unwrap().to_vec(),
+        }
+    }
+
+    pub fn json(&self) -> Option<Value> {
+        serde_json::from_slice(&self.body).ok()
+    }
+
+    pub fn text(&self) -> String {
+        String::from_utf8_lossy(&self.body).to_string()
+    }
+
+    pub fn is_client_error(&self) -> bool {
+        (400..500).contains(&self.status)
+    }
+
+    pub fn content_type(&self) -> Option<&str> {
+        self.headers
+            .get(reqwest::header::CONTENT_TYPE)
+            .and_then(|v| v.to_str().ok())
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum ChangeType<'x> {
+    Created(&'x str),
+    Updated(&'x str),
+    Destroyed(&'x str),
+}
+
+impl Account {
+    pub async fn jmap_get(
+        &self,
+        object: impl Display,
+        properties: impl IntoIterator<Item = impl Display>,
+        ids: impl IntoIterator<Item = impl Display>,
+    ) -> JmapResponse {
+        self.jmap_get_account(self, object, properties, ids).await
+    }
+
+    pub async fn jmap_get_account(
+        &self,
+        account: &Account,
+        object: impl Display,
+        properties: impl IntoIterator<Item = impl Display>,
+        ids: impl IntoIterator<Item = impl Display>,
+    ) -> JmapResponse {
+        let ids = ids
+            .into_iter()
+            .map(|id| Value::String(id.to_string()))
+            .collect::<Vec<Value>>();
+        let properties = properties
+            .into_iter()
+            .map(|p| Value::String(p.to_string()))
+            .collect::<Vec<_>>();
+        let properties = if properties.is_empty() {
+            Value::Null
+        } else {
+            Value::Array(properties)
+        };
+
+        if account.id().document_id() != u32::MAX {
+            self.jmap_method_calls(json!([[
+                format!("{object}/get"),
+                {
+                    "accountId": account.id_string(),
+                    "properties": properties,
+                    "ids": if !ids.is_empty() { Some(ids) } else { None }
+                },
+                "0"
+            ]]))
+            .await
+        } else {
+            self.jmap_method_calls(json!([[
+                format!("{object}/get"),
+                {
+                    "properties": properties,
+                    "ids": if !ids.is_empty() { Some(ids) } else { None }
+                },
+                "0"
+            ]]))
+            .await
+        }
+    }
+
+    pub async fn jmap_query(
+        &self,
+        object: impl Display,
+        filter: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+        sort_by: impl IntoIterator<Item = impl Display>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        let filter = filter
+            .into_iter()
+            .map(|(k, v)| (k.to_string(), v.into()))
+            .collect::<serde_json::Map<_, _>>();
+        let sort_by = sort_by
+            .into_iter()
+            .map(|id| {
+                json! ({
+                    "property": id.to_string()
+                })
+            })
+            .collect::<Vec<Value>>();
+        let arguments = [
+            ("accountId".to_string(), self.id_string().into()),
+            ("filter".to_string(), Value::Object(filter)),
+            ("sort".to_string(), Value::Array(sort_by)),
+        ]
+        .into_iter()
+        .chain(
+            arguments
+                .into_iter()
+                .map(|(k, v)| (k.to_string(), v.into())),
+        )
+        .collect::<serde_json::Map<_, _>>();
+
+        self.jmap_method_calls(json!([[format!("{object}/query"), arguments, "0"]]))
+            .await
+    }
+
+    pub async fn jmap_create(
+        &self,
+        object: impl Display,
+        items: impl IntoIterator<Item = Value>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        self.jmap_create_account(self, object, items, arguments)
+            .await
+    }
+
+    pub async fn jmap_create_account(
+        &self,
+        account: &Account,
+        object: impl Display,
+        items: impl IntoIterator<Item = Value>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        let create = items
+            .into_iter()
+            .enumerate()
+            .map(|(i, item)| (format!("i{i}"), item))
+            .collect::<serde_json::Map<_, _>>();
+        let arguments = [
+            (
+                "accountId".to_string(),
+                Value::String(account.id_string().to_string()),
+            ),
+            ("create".to_string(), Value::Object(create)),
+        ]
+        .into_iter()
+        .chain(
+            arguments
+                .into_iter()
+                .map(|(k, v)| (k.to_string(), v.into())),
+        )
+        .collect::<serde_json::Map<_, _>>();
+
+        self.jmap_method_calls(json!([[format!("{object}/set"), arguments, "0"]]))
+            .await
+    }
+
+    pub async fn jmap_update(
+        &self,
+        object: impl Display,
+        items: impl IntoIterator<Item = (impl Display, Value)>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        self.jmap_update_account(self, object, items, arguments)
+            .await
+    }
+
+    pub async fn jmap_update_account(
+        &self,
+        account: &Account,
+        object: impl Display,
+        items: impl IntoIterator<Item = (impl Display, Value)>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        let update = items
+            .into_iter()
+            .map(|(i, item)| (i.to_string(), item))
+            .collect::<serde_json::Map<_, _>>();
+        let arguments = [
+            (
+                "accountId".to_string(),
+                Value::String(account.id_string().to_string()),
+            ),
+            ("update".to_string(), Value::Object(update)),
+        ]
+        .into_iter()
+        .chain(
+            arguments
+                .into_iter()
+                .map(|(k, v)| (k.to_string(), v.into())),
+        )
+        .collect::<serde_json::Map<_, _>>();
+
+        self.jmap_method_calls(json!([[format!("{object}/set"), arguments, "0"]]))
+            .await
+    }
+
+    pub async fn jmap_destroy(
+        &self,
+        object: impl Display,
+        items: impl IntoIterator<Item = impl Display>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        self.jmap_destroy_account(self, object, items, arguments)
+            .await
+    }
+
+    pub async fn jmap_destroy_account(
+        &self,
+        account: &Account,
+        object: impl Display,
+        items: impl IntoIterator<Item = impl Display>,
+        arguments: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+    ) -> JmapResponse {
+        let destroy = items
+            .into_iter()
+            .map(|id| Value::String(id.to_string()))
+            .collect::<Vec<_>>();
+        let arguments = [
+            (
+                "accountId".to_string(),
+                Value::String(account.id_string().to_string()),
+            ),
+            ("destroy".to_string(), Value::Array(destroy)),
+        ]
+        .into_iter()
+        .chain(
+            arguments
+                .into_iter()
+                .map(|(k, v)| (k.to_string(), v.into())),
+        )
+        .collect::<serde_json::Map<_, _>>();
+
+        self.jmap_method_calls(json!([[format!("{object}/set"), arguments, "0"]]))
+            .await
+    }
+
+    pub async fn jmap_copy(
+        &self,
+        from_account: &Account,
+        to_account: &Account,
+        object: impl Display,
+        items: impl IntoIterator<Item = (impl Display, Value)>,
+        on_success_destroy: bool,
+    ) -> JmapResponse {
+        self.jmap_method_calls(json!([[
+            format!("{object}/copy"),
+            {
+                "fromAccountId": from_account.id_string(),
+                "accountId": to_account.id_string(),
+                "onSuccessDestroyOriginal": on_success_destroy,
+                "create": items
+                        .into_iter()
+                        .map(|(i, item)| (i.to_string(), item)).collect::<serde_json::Map<_, _>>()
+            },
+            "0"
+        ]]))
+        .await
+    }
+
+    pub async fn jmap_changes(&self, object: impl Display, state: impl Display) -> JmapResponse {
+        self.jmap_method_calls(json!([[
+            format!("{object}/changes"),
+            {
+                "accountId": self.id_string(),
+                "sinceState": state.to_string()
+            },
+            "0"
+        ]]))
+        .await
+    }
+
+    pub async fn jmap_method_call(&self, method_name: &str, body: Value) -> JmapResponse {
+        self.jmap_method_calls(json!([[method_name, body, "0"]]))
+            .await
+    }
+
+    pub fn basic_auth(&self) -> String {
+        format!(
+            "Basic {}",
+            general_purpose::STANDARD.encode(format!("{}:{}", self.name(), self.secret()))
+        )
+    }
+
+    pub fn base_url(&self) -> String {
+        format!("https://127.0.0.1:{}", self.http_listener_port)
+    }
+
+    pub fn api_url(&self) -> String {
+        format!("{}/jmap", self.base_url())
+    }
+
+    fn http_client(&self, timeout_ms: u64) -> reqwest::Client {
+        reqwest::Client::builder()
+            .danger_accept_invalid_certs(true)
+            .timeout(Duration::from_millis(timeout_ms))
+            .build()
+            .unwrap()
+    }
+
+    pub async fn http_get_raw(&self, url: &str, accept: Option<&str>) -> RawResponse {
+        let mut req = self
+            .http_client(5000)
+            .get(url)
+            .header(header::AUTHORIZATION, self.basic_auth());
+        if let Some(accept) = accept {
+            req = req.header(header::ACCEPT, accept);
+        }
+        RawResponse::from_response(req.send().await.unwrap()).await
+    }
+
+    pub async fn http_post_raw(
+        &self,
+        url: &str,
+        content_type: &str,
+        body: impl Into<Vec<u8>>,
+    ) -> RawResponse {
+        RawResponse::from_response(
+            self.http_client(5000)
+                .post(url)
+                .header(header::AUTHORIZATION, self.basic_auth())
+                .header(header::CONTENT_TYPE, content_type)
+                .body(body.into())
+                .send()
+                .await
+                .unwrap(),
+        )
+        .await
+    }
+
+    pub async fn jmap_raw_post(&self, body: impl Into<Vec<u8>>, content_type: &str) -> RawResponse {
+        let url = self.api_url();
+        self.http_post_raw(&url, content_type, body).await
+    }
+
+    pub async fn jmap_request(&self, using: &[&str], calls: Value) -> JmapResponse {
+        let body = json!({
+            "using": using,
+            "methodCalls": calls
+        });
+        let raw = self
+            .jmap_raw_post(body.to_string(), "application/json")
+            .await;
+        JmapResponse(
+            raw.json()
+                .unwrap_or_else(|| panic!("Response was not valid JSON: {}", raw.text())),
+        )
+    }
+
+    pub async fn jmap_method_calls(&self, calls: Value) -> JmapResponse {
+        let mut headers = header::HeaderMap::new();
+
+        headers.insert(
+            header::AUTHORIZATION,
+            header::HeaderValue::from_str(&self.basic_auth()).unwrap(),
+        );
+
+        let body = json!({
+          "using": [
+            "urn:ietf:params:jmap:core",
+            "urn:ietf:params:jmap:mail",
+            "urn:ietf:params:jmap:submission",
+            "urn:ietf:params:jmap:vacationresponse",
+            "urn:ietf:params:jmap:contacts",
+            "urn:ietf:params:jmap:contacts:parse",
+            "urn:ietf:params:jmap:calendars",
+            "urn:ietf:params:jmap:calendars:parse",
+            "urn:ietf:params:jmap:websocket",
+            "urn:ietf:params:jmap:sieve",
+            "urn:ietf:params:jmap:blob",
+            "urn:ietf:params:jmap:quota",
+            "urn:ietf:params:jmap:principals",
+            "urn:ietf:params:jmap:principals:owner",
+            "urn:ietf:params:jmap:principals:availability",
+            "urn:ietf:params:jmap:filenode",
+            "urn:ietf:params:jmap:mail:share",
+            "urn:stalwart:jmap"
+          ],
+          "methodCalls": calls
+        });
+
+        JmapResponse(
+            serde_json::from_slice(
+                &reqwest::Client::builder()
+                    .danger_accept_invalid_certs(true)
+                    .timeout(Duration::from_millis(5000))
+                    .default_headers(headers)
+                    .build()
+                    .unwrap()
+                    .post(format!(
+                        "https://127.0.0.1:{}/jmap",
+                        self.http_listener_port
+                    ))
+                    .body(body.to_string())
+                    .send()
+                    .await
+                    .unwrap()
+                    .bytes()
+                    .await
+                    .unwrap(),
+            )
+            .unwrap(),
+        )
+    }
+
+    pub async fn jmap_session_object(&self) -> JmapResponse {
+        let mut headers = header::HeaderMap::new();
+
+        headers.insert(
+            header::AUTHORIZATION,
+            header::HeaderValue::from_str(&format!(
+                "Basic {}",
+                general_purpose::STANDARD.encode(format!("{}:{}", self.name(), self.secret()))
+            ))
+            .unwrap(),
+        );
+
+        JmapResponse(
+            serde_json::from_slice(
+                &reqwest::Client::builder()
+                    .danger_accept_invalid_certs(true)
+                    .timeout(Duration::from_millis(1000))
+                    .default_headers(headers)
+                    .build()
+                    .unwrap()
+                    .get(format!(
+                        "https://127.0.0.1:{}/jmap/session",
+                        self.http_listener_port
+                    ))
+                    .send()
+                    .await
+                    .unwrap()
+                    .bytes()
+                    .await
+                    .unwrap(),
+            )
+            .unwrap(),
+        )
+    }
+
+    pub async fn destroy_all_addressbooks(&self) {
+        self.jmap_method_calls(json!([[
+            "AddressBook/get",
+            {
+              "accountId": self.id_string(),
+              "ids" : (),
+              "properties" : [
+                "id"
+              ]
+            },
+            "R1"
+          ],
+          [
+            "AddressBook/set",
+            {
+              "accountId": self.id_string(),
+              "#destroy" : {
+                    "resultOf": "R1",
+                    "name": "AddressBook/get",
+                    "path": "/list/*/id"
+                },
+              "onDestroyRemoveContents" : true
+            },
+            "R2"
+          ]
+        ]))
+        .await;
+    }
+
+    pub async fn destroy_all_calendars(&self) {
+        self.jmap_method_calls(json!([[
+            "Calendar/get",
+            {
+              "accountId": self.id_string(),
+              "ids" : (),
+              "properties" : [
+                "id"
+              ]
+            },
+            "R1"
+          ],
+          [
+            "Calendar/set",
+            {
+              "accountId": self.id_string(),
+              "#destroy" : {
+                    "resultOf": "R1",
+                    "name": "Calendar/get",
+                    "path": "/list/*/id"
+                },
+              "onDestroyRemoveEvents" : true
+            },
+            "R2"
+          ]
+        ]))
+        .await;
+    }
+
+    pub async fn destroy_all_event_notifications(&self) {
+        self.jmap_method_calls(json!([[
+            "CalendarEventNotification/get",
+            {
+              "accountId": self.id_string(),
+              "ids" : (),
+              "properties" : [
+                "id"
+              ]
+            },
+            "R1"
+          ],
+          [
+            "CalendarEventNotification/set",
+            {
+              "accountId": self.id_string(),
+              "#destroy" : {
+                    "resultOf": "R1",
+                    "name": "CalendarEventNotification/get",
+                    "path": "/list/*/id"
+                }
+            },
+            "R2"
+          ]
+        ]))
+        .await;
+    }
+}
+
+impl JmapResponse {
+    pub fn created(&self, item_idx: u32) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/created/i{item_idx}"))
+            .unwrap_or_else(|| panic!("Missing created item {item_idx}: {self:?}"))
+    }
+
+    pub fn created_id(&self, item_idx: u32) -> Id {
+        Id::from_str(self.created(item_idx).id()).unwrap_or_else(|_| {
+            panic!("Created item {item_idx} does not have a valid id: {self:?}")
+        })
+    }
+
+    pub fn not_created(&self, item_idx: u32) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/notCreated/i{item_idx}"))
+            .unwrap_or_else(|| panic!("Missing not created item {item_idx}: {self:?}"))
+    }
+
+    pub fn updated(&self, id: &str) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/updated/{id}"))
+            .unwrap_or_else(|| panic!("Missing updated item {id}: {self:?}"))
+    }
+
+    pub fn updated_id(&self, id: Id) -> &Value {
+        self.updated(&id.to_string())
+    }
+
+    pub fn not_updated(&self, id: &str) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/notUpdated/{id}"))
+            .unwrap_or_else(|| panic!("Missing not updated item {id}: {self:?}"))
+    }
+
+    pub fn copied(&self, id: &str) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/created/{id}"))
+            .unwrap_or_else(|| panic!("Missing created item {id}: {self:?}"))
+    }
+
+    pub fn method_response(&self) -> &Value {
+        self.0
+            .pointer("/methodResponses/0/1")
+            .unwrap_or_else(|| panic!("Missing method response in response: {self:?}"))
+    }
+
+    pub fn num_responses(&self) -> usize {
+        self.0
+            .pointer("/methodResponses")
+            .and_then(|v| v.as_array())
+            .map(|a| a.len())
+            .unwrap_or(0)
+    }
+
+    pub fn name_at(&self, n: usize) -> &str {
+        self.0
+            .pointer(&format!("/methodResponses/{n}/0"))
+            .and_then(|v| v.as_str())
+            .unwrap_or_else(|| panic!("Missing method name at {n}: {self:?}"))
+    }
+
+    pub fn response_at(&self, n: usize) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/{n}/1"))
+            .unwrap_or_else(|| panic!("Missing method response at {n}: {self:?}"))
+    }
+
+    pub fn call_id_at(&self, n: usize) -> &str {
+        self.0
+            .pointer(&format!("/methodResponses/{n}/2"))
+            .and_then(|v| v.as_str())
+            .unwrap_or_else(|| panic!("Missing call id at {n}: {self:?}"))
+    }
+
+    pub fn is_error_at(&self, n: usize) -> bool {
+        self.0
+            .pointer(&format!("/methodResponses/{n}/0"))
+            .and_then(|v| v.as_str())
+            == Some("error")
+    }
+
+    pub fn error_type_at(&self, n: usize) -> Option<&str> {
+        self.0
+            .pointer(&format!("/methodResponses/{n}/1/type"))
+            .and_then(|v| v.as_str())
+    }
+
+    pub fn session_state(&self) -> Option<&str> {
+        self.0.pointer("/sessionState").and_then(|v| v.as_str())
+    }
+
+    pub fn list_array(&self) -> &Value {
+        self.0
+            .pointer("/methodResponses/0/1/list")
+            .unwrap_or_else(|| panic!("Missing list in response: {self:?}"))
+    }
+
+    pub fn list(&self) -> &[Value] {
+        self.0
+            .pointer("/methodResponses/0/1/list")
+            .and_then(|v| v.as_array())
+            .unwrap_or_else(|| panic!("Missing list in response: {self:?}"))
+    }
+
+    pub fn not_found(&self) -> impl Iterator<Item = &str> {
+        self.0
+            .pointer("/methodResponses/0/1/notFound")
+            .and_then(|v| v.as_array())
+            .unwrap_or_else(|| panic!("Missing notFound in response: {self:?}"))
+            .iter()
+            .map(|v| v.as_str().unwrap())
+    }
+
+    pub fn ids(&self) -> impl Iterator<Item = &str> {
+        self.0
+            .pointer("/methodResponses/0/1/ids")
+            .and_then(|v| v.as_array())
+            .unwrap_or_else(|| panic!("Missing ids in response: {self:?}"))
+            .iter()
+            .map(|v| v.as_str().unwrap())
+    }
+
+    pub fn object_ids(&self) -> impl Iterator<Item = Id> {
+        self.ids().map(move |id| {
+            Id::from_str(id).unwrap_or_else(|_| panic!("Invalid id {id} in response: {self:?}"))
+        })
+    }
+
+    pub fn destroyed(&self) -> impl Iterator<Item = &str> {
+        self.0
+            .pointer("/methodResponses/0/1/destroyed")
+            .and_then(|v| v.as_array())
+            .unwrap_or_else(|| panic!("Missing destroyed in response: {self:?}"))
+            .iter()
+            .map(|v| v.as_str().unwrap())
+    }
+
+    pub fn destroyed_ids(&self) -> impl Iterator<Item = Id> {
+        self.destroyed().map(move |id| {
+            Id::from_str(id).unwrap_or_else(|_| panic!("Invalid id {id} in response: {self:?}"))
+        })
+    }
+
+    pub fn assert_destroyed(&self, expected: &[Id]) -> &Self {
+        let destroyed_ids = self.destroyed_ids().collect::<Vec<_>>();
+        for expected in expected {
+            if !destroyed_ids.contains(expected) {
+                panic!(
+                    "Expected id {expected} to be destroyed but got destroyed ids {destroyed_ids:?}: {self:?}"
+                );
+            }
+        }
+        self
+    }
+
+    pub fn not_destroyed(&self, id: &str) -> &Value {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/notDestroyed/{id}"))
+            .unwrap_or_else(|| panic!("Missing not destroyed item {id}: {self:?}"))
+    }
+
+    pub fn state(&self) -> &str {
+        self.0
+            .pointer("/methodResponses/0/1/state")
+            .and_then(|v| v.as_str())
+            .unwrap_or_else(|| panic!("Missing state in response: {self:?}"))
+    }
+
+    pub fn new_state(&self) -> &str {
+        self.0
+            .pointer("/methodResponses/0/1/newState")
+            .and_then(|v| v.as_str())
+            .unwrap_or_else(|| panic!("Missing new state in response: {self:?}"))
+    }
+
+    pub fn changes(&self) -> impl Iterator<Item = ChangeType<'_>> {
+        self.changes_by_type("created")
+            .map(ChangeType::Created)
+            .chain(self.changes_by_type("updated").map(ChangeType::Updated))
+            .chain(self.changes_by_type("destroyed").map(ChangeType::Destroyed))
+    }
+
+    fn changes_by_type(&self, typ: &str) -> impl Iterator<Item = &str> {
+        self.0
+            .pointer(&format!("/methodResponses/0/1/{typ}"))
+            .and_then(|v| v.as_array())
+            .unwrap_or_else(|| panic!("Missing {typ} changes in response: {self:?}"))
+            .iter()
+            .map(|v| v.as_str().unwrap())
+    }
+
+    pub fn pointer(&self, pointer: &str) -> Option<&Value> {
+        self.0.pointer(pointer)
+    }
+
+    pub fn into_inner(self) -> Value {
+        self.0
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, serde::Deserialize)]
+pub struct JmapSetError {
+    #[serde(rename = "type")]
+    pub type_: SetErrorType,
+
+    #[serde(default)]
+    pub description: Option<String>,
+
+    #[serde(default)]
+    pub properties: Option<Vec<String>>,
+
+    #[serde(rename = "existingId")]
+    #[serde(default)]
+    pub existing_id: Option<Id>,
+
+    #[serde(rename = "objectId")]
+    #[serde(default)]
+    pub object_id: Option<ObjectId>,
+
+    #[serde(default)]
+    #[serde(rename = "linkedObjects")]
+    pub linked_objects: Vec<ObjectId>,
+
+    #[serde(default)]
+    #[serde(rename = "validationErrors")]
+    pub validation_errors: Vec<ValidationError>,
+}
+
+impl JmapSetError {
+    pub fn assert_type(&self, expected: SetErrorType) -> &Self {
+        if self.type_ != expected {
+            panic!("Expected error type {expected:?} but got {self:?}");
+        }
+        self
+    }
+
+    pub fn assert_description_contains(&self, expected: &str) -> &Self {
+        if let Some(description) = &self.description {
+            if !description.contains(expected) {
+                panic!("Expected error description to contain {expected} but got {description}");
+            }
+        } else {
+            panic!("Expected error description to contain {expected} but got no description");
+        }
+        self
+    }
+
+    pub fn assert_properties(&self, expected: &[&str]) -> &Self {
+        let properties = self.properties.as_ref().unwrap_or_else(|| {
+            panic!("Expected error to have properties {expected:?} but got no properties: {self:?}")
+        });
+        for expected in expected {
+            if !properties.contains(&expected.to_string()) {
+                panic!(
+                    "Expected error to have property {expected} but got properties {properties:?}: {self:?}"
+                );
+            }
+        }
+        self
+    }
+}
+
+pub trait JmapUtils {
+    fn id(&self) -> &str {
+        self.text_field("id")
+    }
+
+    fn object_id(&self) -> Id {
+        self.id()
+            .parse()
+            .unwrap_or_else(|_| panic!("Invalid id {} in object", self.id()))
+    }
+
+    fn blob_id(&self) -> &str {
+        self.text_field("blobId")
+    }
+
+    fn typ(&self) -> &str {
+        self.text_field("type")
+    }
+
+    fn description(&self) -> &str {
+        self.text_field("description")
+    }
+
+    fn to_set_error(&self) -> JmapSetError;
+
+    fn with_property(self, field: impl Display, value: impl Into<Value>) -> Self;
+
+    fn text_field(&self, field: &str) -> &str;
+
+    fn integer_field(&self, field: &str) -> i64;
+
+    fn assert_is_equal(&self, other: Value);
+}
+
+impl JmapUtils for Value {
+    fn text_field(&self, field: &str) -> &str {
+        self.pointer(&format!("/{field}"))
+            .and_then(|v| v.as_str())
+            .unwrap_or_else(|| panic!("Missing {field} in object: {self:?}"))
+    }
+
+    fn integer_field(&self, field: &str) -> i64 {
+        self.pointer(&format!("/{field}"))
+            .and_then(|v| v.as_i64())
+            .unwrap_or_else(|| panic!("Missing {field} in object: {self:?}"))
+    }
+
+    fn to_set_error(&self) -> JmapSetError {
+        serde_json::from_str(&self.to_string()).expect("Failed to deserialize set error")
+    }
+
+    fn assert_is_equal(&self, expected: Value) {
+        if self != &expected {
+            panic!(
+                "Values are not equal:\ngot: {}\nexpected: {}",
+                serde_json::to_string_pretty(self).unwrap(),
+                serde_json::to_string_pretty(&expected).unwrap()
+            );
+        }
+    }
+
+    fn with_property(mut self, field: impl Display, value: impl Into<Value>) -> Self {
+        if let Value::Object(map) = &mut self {
+            map.insert(field.to_string(), value.into());
+        } else {
+            panic!("Not an object: {self:?}");
+        }
+        self
+    }
+}
+
+impl<'x> ChangeType<'x> {
+    pub fn as_created(&self) -> &str {
+        match self {
+            ChangeType::Created(id) => id,
+            _ => panic!("Not a created change: {self:?}"),
+        }
+    }
+
+    pub fn as_updated(&self) -> &str {
+        match self {
+            ChangeType::Updated(id) => id,
+            _ => panic!("Not an updated change: {self:?}"),
+        }
+    }
+
+    pub fn as_destroyed(&self) -> &str {
+        match self {
+            ChangeType::Destroyed(id) => id,
+            _ => panic!("Not a destroyed change: {self:?}"),
+        }
+    }
+}
+
+impl Display for JmapResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.0, f)
+    }
+}
+
+impl std::fmt::Debug for JmapResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        serde_json::to_string_pretty(&self.0)
+            .map_err(|_| std::fmt::Error)
+            .and_then(|s| std::fmt::Display::fmt(&s, f))
+    }
+}
+
+pub trait IntoJmapSet {
+    fn into_jmap_set(self) -> Value;
+}
+
+impl<T: IntoIterator<Item = impl Display>> IntoJmapSet for T {
+    fn into_jmap_set(self) -> Value {
+        Value::Object(
+            self.into_iter()
+                .map(|id| (id.to_string(), Value::Bool(true)))
+                .collect::<serde_json::Map<String, Value>>(),
+        )
+    }
+}

tests/src/utils/mod.rs

@@ -0,0 +1,22 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod account;
+pub mod cleanup;
+pub mod containers;
+pub mod dns;
+pub mod http;
+pub mod http_server;
+pub mod imap;
+pub mod jmap;
+pub mod pop3;
+pub mod registry;
+pub mod server;
+pub mod sieve;
+pub mod smtp;
+pub mod storage;
+pub mod temp_dir;
+pub mod webdav;

tests/src/utils/pop3.rs

@@ -0,0 +1,105 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use base64::{Engine, engine::general_purpose};
+use rustls_pki_types::ServerName;
+use std::time::Duration;
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+use tokio_rustls::client::TlsStream;
+use utils::tls::build_tls_connector;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ResponseType {
+    Ok,
+    Multiline,
+    Err,
+}
+
+pub struct Pop3Connection {
+    reader: Lines<BufReader<ReadHalf<TlsStream<TcpStream>>>>,
+    writer: WriteHalf<TlsStream<TcpStream>>,
+}
+
+impl Pop3Connection {
+    pub async fn connect() -> Self {
+        let (reader, writer) = tokio::io::split(
+            build_tls_connector(true)
+                .unwrap()
+                .connect(
+                    ServerName::try_from("pop3.example.org").unwrap().to_owned(),
+                    TcpStream::connect("127.0.0.1:4110").await.unwrap(),
+                )
+                .await
+                .unwrap(),
+        );
+
+        let mut conn = Pop3Connection {
+            reader: BufReader::new(reader).lines(),
+            writer,
+        };
+
+        conn.assert_read(ResponseType::Ok).await;
+        conn
+    }
+
+    pub async fn authenticate(&mut self, user: &str, pass: &str) {
+        let creds = general_purpose::STANDARD.encode(format!("\0{user}\0{pass}"));
+        self.send(&format!("AUTH PLAIN {creds}")).await;
+        self.assert_read(ResponseType::Ok).await;
+    }
+
+    pub async fn assert_read(&mut self, rt: ResponseType) -> Vec<String> {
+        let lines = self.read(matches!(rt, ResponseType::Multiline)).await;
+        if lines.last().unwrap().starts_with(match rt {
+            ResponseType::Ok => "+OK",
+            ResponseType::Multiline => ".",
+            ResponseType::Err => "-ERR",
+        }) {
+            lines
+        } else {
+            panic!("Expected {:?} from server but got: {:?}", rt, lines);
+        }
+    }
+
+    pub async fn read(&mut self, is_multiline: bool) -> Vec<String> {
+        let mut lines = Vec::new();
+        loop {
+            match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+                Ok(Ok(Some(line))) => {
+                    let is_done = (!is_multiline && line.starts_with("+OK"))
+                        || (is_multiline && line == ".")
+                        || line.starts_with("-ERR");
+                    //let c = println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+}

tests/src/utils/registry.rs

@@ -0,0 +1,470 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::utils::{
+    account::Account,
+    jmap::{JmapResponse, JmapSetError, JmapUtils},
+};
+use registry::{
+    schema::{
+        prelude::{ObjectType, Property},
+        structs::{
+            Action, Expression, MtaExtensions, MtaStageAuth, MtaStageData, MtaStageEhlo,
+            MtaStageRcpt, SpamSettings,
+        },
+    },
+    types::{EnumImpl, ObjectImpl},
+};
+use serde_json::{Value, json};
+use std::fmt::Display;
+use store::registry::write::RegistryWriteResult;
+use types::id::Id;
+
+impl Account {
+    pub async fn registry_create<T: ObjectImpl>(
+        &self,
+        items: impl IntoIterator<Item = T>,
+    ) -> JmapResponse {
+        let typ = T::OBJECT;
+        let name = typ.as_str();
+
+        self.jmap_create_account(
+            self,
+            format!("x:{name}"),
+            items.into_iter().map(|item| {
+                let mut item =
+                    serde_json::to_value(item).expect("Failed to serialize item to JSON");
+                remove_server_set_props(typ, &mut item);
+                item
+            }),
+            Vec::<(&str, &str)>::new(),
+        )
+        .await
+    }
+
+    pub async fn registry_create_many(
+        &self,
+        object_type: ObjectType,
+        items: impl IntoIterator<Item = Value>,
+    ) -> JmapResponse {
+        let name = object_type.as_str();
+        self.jmap_create_account(self, format!("x:{name}"), items, Vec::<(&str, &str)>::new())
+            .await
+    }
+
+    pub async fn registry_get<T: ObjectImpl>(&self, id: Id) -> T {
+        let name = T::OBJECT.as_str();
+
+        let value = self
+            .jmap_get_account(self, format!("x:{name}"), Vec::<&str>::new(), vec![id])
+            .await
+            .list()[0]
+            .to_string();
+        serde_json::from_str(&value).unwrap_or_else(|_| {
+            panic!("Failed to deserialize {value}");
+        })
+    }
+
+    pub async fn registry_get_all<T: ObjectImpl>(&self) -> Vec<(Id, T)> {
+        let name = T::OBJECT.as_str();
+
+        let response = self
+            .jmap_get_account(
+                self,
+                format!("x:{name}"),
+                Vec::<&str>::new(),
+                Vec::<Id>::new(),
+            )
+            .await;
+        let mut items = Vec::with_capacity(response.list().len());
+        for item in response.list() {
+            let id = item.object_id();
+            let item = serde_json::from_str(&item.to_string()).unwrap_or_else(|err| {
+                panic!("Failed to deserialize {item} : {err}");
+            });
+            items.push((id, item));
+        }
+        items
+    }
+
+    pub async fn registry_get_many(
+        &self,
+        object_type: ObjectType,
+        ids: impl IntoIterator<Item = impl Display>,
+    ) -> JmapResponse {
+        self.jmap_get_account(
+            self,
+            format!("x:{}", object_type.as_str()),
+            Vec::<&str>::new(),
+            ids,
+        )
+        .await
+    }
+
+    pub async fn registry_update(
+        &self,
+        object: ObjectType,
+        items: impl IntoIterator<Item = (impl Display, Value)>,
+    ) -> JmapResponse {
+        let name = object.as_str();
+
+        self.jmap_update_account(self, format!("x:{name}"), items, Vec::<(&str, &str)>::new())
+            .await
+    }
+
+    pub async fn registry_query_ids(
+        &self,
+        object: ObjectType,
+        filter: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+        sort_by: impl IntoIterator<Item = impl Display>,
+    ) -> Vec<Id> {
+        self.registry_query(object, filter, sort_by)
+            .await
+            .object_ids()
+            .collect()
+    }
+
+    pub async fn registry_query(
+        &self,
+        object: ObjectType,
+        filter: impl IntoIterator<Item = (impl Display, impl Into<Value>)>,
+        sort_by: impl IntoIterator<Item = impl Display>,
+    ) -> JmapResponse {
+        let name = object.as_str();
+
+        self.jmap_query(
+            format!("x:{name}"),
+            filter,
+            sort_by,
+            Vec::<(&str, &str)>::new(),
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub async fn registry_query_paginated(
+        &self,
+        object: ObjectType,
+        sort_property: &str,
+        sort_ascending: bool,
+        position: Option<i32>,
+        limit: Option<usize>,
+        anchor: Option<Id>,
+        anchor_offset: Option<i32>,
+        calculate_total: bool,
+    ) -> JmapResponse {
+        let name = object.as_str();
+        let mut args = serde_json::Map::new();
+        args.insert("filter".into(), json!({}));
+        args.insert(
+            "sort".into(),
+            json!([{ "property": sort_property, "isAscending": sort_ascending }]),
+        );
+        if let Some(p) = position {
+            args.insert("position".into(), json!(p));
+        }
+        if let Some(l) = limit {
+            args.insert("limit".into(), json!(l));
+        }
+        if let Some(a) = anchor {
+            args.insert("anchor".into(), json!(a.to_string()));
+        }
+        if let Some(ao) = anchor_offset {
+            args.insert("anchorOffset".into(), json!(ao));
+        }
+        if calculate_total {
+            args.insert("calculateTotal".into(), json!(true));
+        }
+
+        self.jmap_method_calls(json!([[
+            format!("x:{name}/query"),
+            Value::Object(args),
+            "0"
+        ]]))
+        .await
+    }
+
+    pub async fn registry_destroy(
+        &self,
+        object: ObjectType,
+        items: impl IntoIterator<Item = impl Display>,
+    ) -> JmapResponse {
+        let name = object.as_str();
+
+        self.jmap_destroy_account(self, format!("x:{name}"), items, Vec::<(&str, &str)>::new())
+            .await
+    }
+
+    pub async fn registry_destroy_all(&self, object: ObjectType) {
+        let name = object.as_str();
+        self.jmap_method_calls(json!([[
+            format!("x:{name}/get"),
+            {
+              "ids" : (),
+              "properties" : [
+                "id"
+              ]
+            },
+            "R1"
+          ],
+          [
+            format!("x:{name}/set"),
+            {
+              "#destroy" : {
+                    "resultOf": "R1",
+                    "name": format!("x:{name}/get"),
+                    "path": "/list/*/id"
+                },
+            },
+            "R2"
+          ]
+        ]))
+        .await;
+    }
+
+    pub async fn registry_create_object<T: ObjectImpl>(&self, item: T) -> Id {
+        self.registry_create([item]).await.created_id(0)
+    }
+
+    pub async fn registry_create_object_expect_err<T: ObjectImpl>(&self, item: T) -> JmapSetError {
+        self.registry_create([item])
+            .await
+            .not_created(0)
+            .to_set_error()
+    }
+
+    pub async fn registry_update_object(&self, object: ObjectType, id: Id, item: Value) {
+        self.registry_update(object, [(id, item)])
+            .await
+            .updated_id(id);
+    }
+
+    pub async fn registry_update_setting<T: ObjectImpl>(
+        &self,
+        setting: T,
+        properties: &[Property],
+    ) {
+        let mut item = serde_json::to_value(setting).expect("Failed to serialize setting to JSON");
+
+        if !properties.is_empty() {
+            // Only include the specified properties in the update
+            if let Value::Object(obj) = &mut item {
+                obj.retain(|k, _| properties.iter().any(|p| p.as_str() == k));
+            }
+        }
+
+        self.registry_update(T::OBJECT, [(Id::singleton(), item)])
+            .await
+            .updated_id(Id::singleton());
+    }
+
+    pub async fn reload_settings(&self) {
+        self.registry_create_object(Action::ReloadSettings).await;
+    }
+
+    pub async fn reload_lookup_stores(&self) {
+        self.registry_create_object(Action::ReloadLookupStores)
+            .await;
+    }
+
+    pub async fn registry_update_object_expect_err(
+        &self,
+        object: ObjectType,
+        id: Id,
+        item: Value,
+    ) -> JmapSetError {
+        self.registry_update(object, [(id, item)])
+            .await
+            .not_updated(&id.to_string())
+            .to_set_error()
+    }
+
+    pub async fn registry_destroy_object_expect_err(
+        &self,
+        object: ObjectType,
+        id: Id,
+    ) -> JmapSetError {
+        self.registry_destroy(object, [id])
+            .await
+            .not_destroyed(&id.to_string())
+            .to_set_error()
+    }
+
+    pub async fn destroy_account(&self, account: Account) {
+        let account_id = account.id();
+        self.registry_destroy(ObjectType::Account, [account_id])
+            .await
+            .assert_destroyed(&[account_id]);
+    }
+
+    pub async fn mta_allow_relaying(&self) {
+        self.registry_create_object(MtaStageRcpt {
+            allow_relaying: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            ..Default::default()
+        })
+        .await;
+    }
+
+    pub async fn mta_disable_spam_filter(&self) {
+        self.registry_create_object(SpamSettings {
+            enable: false,
+            ..Default::default()
+        })
+        .await;
+    }
+
+    pub async fn mta_no_auth(&self) {
+        self.registry_create_object(MtaStageAuth {
+            require: Expression {
+                else_: "false".into(),
+                ..Default::default()
+            },
+            ..Default::default()
+        })
+        .await;
+    }
+
+    pub async fn mta_all_extensions(&self) {
+        self.registry_create_object(MtaExtensions {
+            chunking: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            deliver_by: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            dsn: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            expn: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            future_release: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            mt_priority: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            no_soliciting: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            pipelining: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            require_tls: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            vrfy: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+        })
+        .await;
+    }
+
+    pub async fn mta_allow_non_fqdn(&self) {
+        self.registry_create_object(MtaStageEhlo {
+            reject_non_fqdn: Expression {
+                else_: "false".into(),
+                ..Default::default()
+            },
+            ..Default::default()
+        })
+        .await;
+    }
+
+    pub async fn mta_add_all_headers(&self) {
+        self.registry_create_object(MtaStageData {
+            add_date_header: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            add_message_id_header: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            add_received_header: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            add_received_spf_header: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            add_auth_results_header: Expression {
+                else_: "true".into(),
+                ..Default::default()
+            },
+            add_return_path_header: Expression {
+                else_: "false".into(),
+                ..Default::default()
+            },
+            enable_spam_filter: Expression {
+                else_: "false".into(),
+                ..Default::default()
+            },
+            ..Default::default()
+        })
+        .await;
+    }
+}
+
+impl JmapResponse {
+    pub fn objects<T: ObjectImpl>(&self) -> impl Iterator<Item = T> {
+        self.list()
+            .iter()
+            .map(|item| serde_json::from_value(item.clone()).expect("Failed to deserialize item"))
+    }
+}
+
+pub trait UnwrapRegistryId {
+    fn unwrap_id(self, location: &str) -> Id;
+}
+
+impl UnwrapRegistryId for RegistryWriteResult {
+    fn unwrap_id(self, location: &str) -> Id {
+        match self {
+            RegistryWriteResult::Success(id) => id,
+            err => panic!("Expected success at {location} but got {err}"),
+        }
+    }
+}
+
+fn remove_server_set_props(typ: ObjectType, value: &mut serde_json::Value) {
+    if let Value::Object(obj) = value {
+        let is_app_pass = matches!(typ, ObjectType::AppPassword | ObjectType::ApiKey)
+            || obj
+                .get("@type")
+                .and_then(|v| v.as_str())
+                .is_some_and(|t| ["AppPassword", "ApiKey"].contains(&t));
+        obj.retain(|k, v| {
+            !([
+                "createdAt",
+                "credentialId",
+                "retireAt",
+                "accountKey",
+                "accountUri",
+            ]
+            .contains(&k.as_str())
+                || (is_app_pass && k == "secret")
+                || (k == "memberTenantId" && v.is_null()))
+        });
+        for v in obj.values_mut() {
+            remove_server_set_props(typ, v);
+        }
+    }
+}

tests/src/utils/server.rs

@@ -0,0 +1,622 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    AssertConfig,
+    smtp::session::{DummyIo, TestSession},
+    utils::{
+        account::Account,
+        cleanup::{search_store_destroy, store_blob_expire_all, store_destroy},
+        registry::UnwrapRegistryId,
+        storage::{RegistryEnvStores, assert_is_empty, build_data_store, wait_for_tasks},
+        temp_dir::TempDir,
+    },
+};
+use ahash::AHashMap;
+use common::{
+    BuildServer, Caches, Core, Data, DavResources, Inner, Server,
+    auth::RECOVERY_ADMIN_ID,
+    config::{
+        server::{Listeners, ServerProtocol},
+        storage::Storage,
+        telemetry::Telemetry,
+    },
+    ipc::{QueueEvent, ReportingEvent},
+    manager::{
+        boot::{IpcReceivers, build_ipc},
+        defaults::BootstrapDefaults,
+    },
+    psl,
+};
+use email::message::metadata::MessageMetadata;
+use groupware::cache::GroupwareCache;
+use http::HttpSessionManager;
+use imap::core::ImapSessionManager;
+use jmap_client::client::Client;
+use managesieve::core::ManageSieveSessionManager;
+use pop3::Pop3SessionManager;
+use registry::{
+    schema::{
+        enums::{EventPolicy, NetworkListenerProtocol, TracingLevel},
+        prelude::{Object, ObjectType, SocketAddr},
+        structs::{
+            Authentication, Certificate, Domain, NetworkListener, PublicText, SecretKeyFile,
+            SecretText, SystemSettings, Tracer, TracerStdout,
+        },
+    },
+    types::{EnumImpl, datetime::UTCDateTime, map::Map},
+};
+use services::{SpawnServices, broadcast::subscriber::spawn_broadcast_subscriber};
+use smtp::{
+    SpawnQueueManager,
+    core::{Session, SmtpSessionManager},
+    queue::{
+        manager::{Queue, SpawnQueue},
+        spool::{QueuedMessages, SmtpSpool},
+    },
+    reporting::scheduler::SpawnReport,
+};
+use std::{collections::VecDeque, path::PathBuf, str::FromStr, sync::Arc};
+use store::{
+    RegistryStore, Store, ValueKey,
+    registry::{RegistryQuery, bootstrap::Bootstrap, write::RegistryWrite},
+    write::{AlignedBytes, Archive, now},
+};
+use tokio::sync::{mpsc, watch};
+use trc::EventType;
+use types::{collection::Collection, field::EmailField, id::Id};
+
+pub struct TestServer {
+    pub server: Server,
+    pub accounts: AHashMap<&'static str, Account>,
+    pub temp_dir: TempDir,
+    pub queue_rx: mpsc::Receiver<QueueEvent>,
+    pub queue_events: VecDeque<QueueEvent>,
+    pub report_rx: mpsc::Receiver<ReportingEvent>,
+    shutdown_tx: watch::Sender<bool>,
+    reset: bool,
+}
+
+pub struct TestServerBuilder {
+    bootstrap: Bootstrap,
+    temp_dir: TempDir,
+    http_listener_port: u16,
+    reset: bool,
+    logging_enabled: bool,
+    capture_queue: bool,
+    capture_reporting: bool,
+    disable_services: bool,
+}
+
+impl TestServerBuilder {
+    pub async fn new(test_name: &str) -> Self {
+        let reset = std::env::var("NO_INSERT").is_err();
+
+        Self::new_with_role(test_name, "mail.example.org".to_string(), None, reset).await
+    }
+
+    pub async fn new_with_role(
+        test_name: &str,
+        hostname: String,
+        node_role: Option<String>,
+        reset: bool,
+    ) -> Self {
+        let temp_dir = TempDir::new(test_name, reset);
+        let path = temp_dir.path.to_string_lossy().to_string();
+        let store_type = std::env::var("STORE").expect(concat!(
+            "Missing or invalid store type. Try ",
+            "running `STORE=<store_type> cargo test`"
+        ));
+        let data_store = build_data_store(&store_type, &path).await;
+        let store = Store::build(data_store).await.unwrap();
+
+        store.create_tables().await.unwrap();
+
+        // Delete old store if requested
+        if reset {
+            store_destroy(&store).await;
+        }
+
+        Self {
+            bootstrap: Bootstrap::new(
+                RegistryStore::new(&path, store, hostname, 1, node_role).await,
+            )
+            .await,
+            http_listener_port: 8899,
+            temp_dir,
+            reset,
+            logging_enabled: false,
+            capture_queue: false,
+            capture_reporting: false,
+            disable_services: false,
+        }
+    }
+
+    pub async fn with_default_listeners(self) -> Self {
+        let mut this = self;
+        for (protocol, name, port, use_tls) in [
+            (NetworkListenerProtocol::Http, "jmap", 8899, true),
+            (NetworkListenerProtocol::Imap, "imap", 9991, false),
+            (NetworkListenerProtocol::Imap, "imaptls", 9992, true),
+            (NetworkListenerProtocol::ManageSieve, "sieve", 4190, true),
+            (NetworkListenerProtocol::Pop3, "pop3", 4110, true),
+            (NetworkListenerProtocol::Lmtp, "lmtp-debug", 11200, false),
+        ] {
+            this = this.with_listener(protocol, name, port, use_tls).await;
+        }
+        this
+    }
+
+    pub async fn with_http_listener(self, port: u16) -> Self {
+        self.with_listener(NetworkListenerProtocol::Http, "jmap", port, true)
+            .await
+    }
+
+    pub async fn with_smtp_listener(self, port: u16) -> Self {
+        self.with_listener(NetworkListenerProtocol::Smtp, "smtp", port, false)
+            .await
+    }
+
+    pub async fn with_imap_listener(self, port: u16) -> Self {
+        self.with_listener(NetworkListenerProtocol::Imap, "imap", port, false)
+            .await
+    }
+
+    pub async fn with_dummy_tls_cert(self, sans: impl IntoIterator<Item = &str>) -> Self {
+        let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        cert_path.push("resources");
+        let mut cert = cert_path.clone();
+        cert.push("tls_cert.pem");
+        let mut pk = cert_path.clone();
+        pk.push("tls_privatekey.pem");
+
+        self.with_object(Certificate {
+            private_key: SecretText::File(SecretKeyFile {
+                file_path: pk.to_string_lossy().to_string(),
+            }),
+            certificate: PublicText::File(SecretKeyFile {
+                file_path: cert.to_string_lossy().to_string(),
+            }),
+            issuer: "Stalwart Test CA".to_string(),
+            not_valid_after: UTCDateTime::from_timestamp((now() + 86400) as i64),
+            subject_alternative_names: Map::new(
+                sans.into_iter().map(|san| san.to_string()).collect(),
+            ),
+            ..Default::default()
+        })
+        .await
+    }
+
+    pub async fn with_listener(
+        mut self,
+        protocol: NetworkListenerProtocol,
+        name: &str,
+        port: u16,
+        tls_implicit: bool,
+    ) -> Self {
+        if protocol == NetworkListenerProtocol::Http {
+            self.http_listener_port = port;
+        }
+        self.insert_object(NetworkListener {
+            bind: Map::new(vec![
+                SocketAddr::from_str(&format!("0.0.0.0:{port}")).unwrap(),
+            ]),
+            name: name.to_string(),
+            protocol,
+            use_tls: true,
+            tls_implicit,
+            ..Default::default()
+        })
+        .await;
+        self
+    }
+
+    pub async fn with_object(self, object: impl Into<Object>) -> Self {
+        self.insert_object(object).await;
+        self
+    }
+
+    pub fn with_logging(mut self) -> Self {
+        self.logging_enabled = true;
+        self
+    }
+
+    pub fn capture_queue(mut self) -> Self {
+        self.capture_queue = true;
+        self
+    }
+
+    pub fn capture_reporting(mut self) -> Self {
+        self.capture_reporting = true;
+        self
+    }
+
+    pub fn disable_services(mut self) -> Self {
+        self.disable_services = true;
+        self
+    }
+
+    pub async fn insert_object(&self, object: impl Into<Object>) -> Id {
+        self.bootstrap
+            .registry
+            .write(RegistryWrite::insert(&object.into()))
+            .await
+            .unwrap()
+            .unwrap_id(trc::location!())
+    }
+
+    pub async fn build(self) -> TestServer {
+        self.build_with_opts(true).await
+    }
+
+    pub async fn build_with_opts(mut self, init_store: bool) -> TestServer {
+        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+
+        if init_store {
+            // Register stores from environment
+            self.bootstrap.registry.insert_stores_from_env().await;
+
+            // Enable logging if requested
+            let level = std::env::var("LOG")
+                .map(|log| TracingLevel::parse(&log).expect("Invalid log level"))
+                .ok();
+
+            // Add default domain
+            let default_domain = psl::domain_str(self.bootstrap.registry.local_hostname()).unwrap();
+            let default_domain_id = self
+                .insert_object(Domain {
+                    name: default_domain.to_string(),
+                    ..Default::default()
+                })
+                .await;
+            self.insert_object(SystemSettings {
+                default_hostname: self.bootstrap.registry.local_hostname().to_string(),
+                default_domain_id,
+                ..Default::default()
+            })
+            .await;
+
+            self.insert_object(Tracer::Stdout(TracerStdout {
+                enable: level.is_some() || self.logging_enabled,
+                level: level.unwrap_or(TracingLevel::Info),
+                ansi: true,
+                multiline: false,
+                events: Map::new(
+                    EventType::variants()
+                        .iter()
+                        .filter(|ev| {
+                            let ev = ev.as_str();
+                            ev.starts_with("network.")
+                                || ev.starts_with("http.connection-")
+                                || ev == "telemetry.webhook-error"
+                                || ev == "http.request-body"
+                                || ev == "http.request-url"
+                                || ev == "tls.no-certificates-available"
+                                || ev == "store.cache-hit"
+                        })
+                        .copied()
+                        .collect(),
+                ),
+                events_policy: EventPolicy::Exclude,
+                ..Default::default()
+            }))
+            .await;
+        }
+
+        // Start listeners
+        let mut servers = Listeners::parse(&mut self.bootstrap).await;
+        servers.bind_and_drop_priv(&mut self.bootstrap);
+
+        // Set HTTP port
+        self.bootstrap.registry = self
+            .bootstrap
+            .registry
+            .clone_with_public_url(format!("https://127.0.0.1:{}", self.http_listener_port));
+
+        if init_store {
+            // Add safe defaults if missing
+            self.bootstrap.insert_safe_defaults().await;
+
+            // Add directory
+            if let Some(directory_id) = self
+                .bootstrap
+                .registry
+                .query::<Vec<Id>>(RegistryQuery::new(ObjectType::Directory))
+                .await
+                .unwrap()
+                .first()
+            {
+                let mut auth = self
+                    .bootstrap
+                    .registry
+                    .object::<Authentication>(Id::singleton())
+                    .await
+                    .unwrap()
+                    .unwrap();
+                auth.directory_id = Some(*directory_id);
+                self.bootstrap
+                    .registry
+                    .write(RegistryWrite::insert(&auth.into()))
+                    .await
+                    .unwrap();
+            }
+        }
+
+        // Parse storage
+        let storage = Storage::parse(&mut self.bootstrap).await;
+
+        // Reset search store
+        if init_store && self.reset {
+            search_store_destroy(&storage.search).await;
+        }
+
+        // Parse telemetry
+        let telemetry = Telemetry::parse(&mut self.bootstrap, &storage).await;
+
+        // Parse components
+        let core = Box::pin(Core::parse(&mut self.bootstrap, storage)).await;
+        let data = Data::parse(&mut self.bootstrap).await;
+        let cache = Caches::parse(&mut self.bootstrap).await;
+
+        // Enable telemetry
+        telemetry.enable(true);
+
+        // Build inner
+        let (ipc, mut ipc_rxs) = build_ipc(!core.storage.coordinator.is_none());
+        let inner = Arc::new(Inner {
+            shared_core: core.into_shared(),
+            data,
+            ipc,
+            cache,
+        });
+
+        // Parse TCP acceptors
+        servers
+            .parse_tcp_acceptors(&mut self.bootstrap, inner.clone())
+            .await;
+
+        // Start services
+        self.bootstrap.assert_no_errors();
+        if !self.disable_services {
+            ipc_rxs.spawn_services(inner.clone());
+        }
+
+        // Spawn queue manager if not capturing
+        let (_, mut queue_rx) = mpsc::channel(100);
+        let (_, mut report_rx) = mpsc::channel(100);
+        if !self.capture_queue && !self.capture_reporting {
+            ipc_rxs.spawn_queue_manager(inner.clone());
+        } else {
+            let queue_rx_ = ipc_rxs.queue_rx.take().unwrap();
+            let report_rx_ = ipc_rxs.report_rx.take().unwrap();
+            if !self.capture_queue {
+                queue_rx_.spawn(inner.clone());
+            } else {
+                queue_rx = queue_rx_;
+            }
+            if !self.capture_reporting {
+                report_rx_.spawn(inner.clone());
+            } else {
+                report_rx = report_rx_;
+            }
+        }
+
+        // Spawn servers
+        let (shutdown_tx, shutdown_rx) = servers.spawn(|server, acceptor, shutdown_rx| {
+            match &server.protocol {
+                ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn(
+                    SmtpSessionManager::new(inner.clone()),
+                    inner.clone(),
+                    acceptor,
+                    shutdown_rx,
+                ),
+                ServerProtocol::Http => server.spawn(
+                    HttpSessionManager::new(inner.clone()),
+                    inner.clone(),
+                    acceptor,
+                    shutdown_rx,
+                ),
+                ServerProtocol::Imap => server.spawn(
+                    ImapSessionManager::new(inner.clone()),
+                    inner.clone(),
+                    acceptor,
+                    shutdown_rx,
+                ),
+                ServerProtocol::Pop3 => server.spawn(
+                    Pop3SessionManager::new(inner.clone()),
+                    inner.clone(),
+                    acceptor,
+                    shutdown_rx,
+                ),
+                ServerProtocol::ManageSieve => server.spawn(
+                    ManageSieveSessionManager::new(inner.clone()),
+                    inner.clone(),
+                    acceptor,
+                    shutdown_rx,
+                ),
+            };
+        });
+
+        // Start broadcast subscriber
+        if !self.disable_services {
+            spawn_broadcast_subscriber(inner.clone(), shutdown_rx);
+        }
+
+        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+
+        let mut admin = Account::new(
+            "admin",
+            "popolna_zapora",
+            &[],
+            "Recovery Admin",
+            Id::from(RECOVERY_ADMIN_ID),
+        );
+        admin.http_listener_port = self.http_listener_port;
+
+        TestServer {
+            server: inner.build_server(),
+            temp_dir: self.temp_dir,
+            accounts: AHashMap::from_iter([("admin", admin)]),
+            queue_rx,
+            queue_events: VecDeque::new(),
+            report_rx,
+            shutdown_tx,
+            reset: self.reset,
+        }
+    }
+}
+
+impl TestServer {
+    pub fn reload_core(&mut self) {
+        self.server = self.server.inner.build_server();
+    }
+
+    pub fn account(&self, name: &str) -> &Account {
+        self.accounts.get(name).unwrap()
+    }
+
+    pub async fn wait_for_tasks(&self) {
+        wait_for_tasks(&self.server, false, false).await;
+    }
+
+    pub async fn wait_for_tasks_skip_failures(&self) {
+        wait_for_tasks(&self.server, false, true).await;
+    }
+
+    pub async fn wait_for_tasks_skip_not_due(&self) {
+        wait_for_tasks(&self.server, true, false).await;
+    }
+
+    pub async fn blob_expire_all(&self) {
+        store_blob_expire_all(&self.server.core.storage.data).await;
+    }
+
+    pub async fn assert_is_empty(&self) {
+        assert_is_empty(&self.server, true).await;
+    }
+
+    pub async fn cleanup(&self) {
+        self.assert_is_empty().await;
+        self.server.invalidate_all_local_caches();
+    }
+
+    pub async fn destroy_store(&self) {
+        store_destroy(self.server.store()).await;
+    }
+
+    pub fn is_reset(&self) -> bool {
+        self.reset
+    }
+
+    pub fn tmp_dir(&self) -> &str {
+        self.temp_dir.path.as_os_str().to_str().unwrap()
+    }
+
+    pub fn shutdown(&self) {
+        let _ = self.shutdown_tx.send(true);
+    }
+
+    pub fn new_mta_session(&self) -> Session<DummyIo> {
+        Session::test(self.server.clone())
+    }
+
+    pub fn new_mta_session_with_shutdown(&self) -> (Session<DummyIo>, watch::Sender<bool>) {
+        let (tx, rx) = watch::channel(true);
+        (Session::test_with_shutdown(self.server.clone(), rx), tx)
+    }
+
+    pub async fn resources(&self, name: &'static str, collection: Collection) -> Arc<DavResources> {
+        let account_id = self.account(name).id().document_id();
+        self.server
+            .fetch_dav_resources(account_id, account_id, collection.into())
+            .await
+            .unwrap()
+    }
+
+    pub async fn fetch_email(&self, account_id: u32, document_id: u32) -> Vec<u8> {
+        let metadata_ = self
+            .server
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::property(
+                account_id,
+                Collection::Email,
+                document_id,
+                EmailField::Metadata,
+            ))
+            .await
+            .unwrap()
+            .unwrap();
+        self.server
+            .blob_store()
+            .get_blob(
+                metadata_
+                    .unarchive::<MessageMetadata>()
+                    .unwrap()
+                    .blob_hash
+                    .0
+                    .as_slice(),
+                0..usize::MAX,
+            )
+            .await
+            .unwrap()
+            .unwrap()
+    }
+
+    pub async fn all_queued_messages(&self) -> QueuedMessages {
+        self.server
+            .next_event(&mut Queue::new(
+                self.server.inner.clone(),
+                mpsc::channel(100).1,
+            ))
+            .await
+    }
+
+    pub async fn destroy_all_mailboxes(&self, account: &Account) {
+        self.wait_for_tasks().await;
+        account.jmap_client().await.destroy_all_mailboxes().await;
+    }
+
+    pub async fn inner_with_rxs(&self) -> (Arc<Inner>, IpcReceivers) {
+        let (ipc, ipc_rxs) = build_ipc(false);
+
+        let mut bp = Bootstrap::new_uninitialized(self.server.registry().clone());
+
+        (
+            Inner {
+                shared_core: self.server.core.as_ref().clone().into_shared(),
+                data: Default::default(),
+                ipc,
+                cache: Caches::parse(&mut bp).await,
+            }
+            .into(),
+            ipc_rxs,
+        )
+    }
+}
+
+impl Account {
+    pub async fn destroy_all_mailboxes_for_account(&self, account_id: u32) {
+        let mut client = self.jmap_client().await;
+        client.set_default_account_id(Id::from(account_id));
+        client.destroy_all_mailboxes().await;
+    }
+}
+
+pub trait DestroyAllMailboxes {
+    fn destroy_all_mailboxes(&self) -> impl Future<Output = ()>;
+}
+
+impl DestroyAllMailboxes for Client {
+    async fn destroy_all_mailboxes(&self) {
+        let mut request = self.build();
+        request.query_mailbox().arguments().sort_as_tree(true);
+        let mut ids = request.send_query_mailbox().await.unwrap().take_ids();
+        ids.reverse();
+        for id in ids {
+            self.mailbox_destroy(&id, true).await.unwrap();
+        }
+    }
+}

tests/src/utils/sieve.rs

@@ -0,0 +1,105 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use base64::{Engine, engine::general_purpose};
+use imap_proto::ResponseType;
+use rustls_pki_types::ServerName;
+use std::time::Duration;
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+use tokio_rustls::client::TlsStream;
+use utils::tls::build_tls_connector;
+
+pub struct SieveConnection {
+    reader: Lines<BufReader<ReadHalf<TlsStream<TcpStream>>>>,
+    writer: WriteHalf<TlsStream<TcpStream>>,
+}
+
+impl SieveConnection {
+    pub async fn connect() -> Self {
+        let (reader, writer) = tokio::io::split(
+            build_tls_connector(true)
+                .unwrap()
+                .connect(
+                    ServerName::try_from("imap.example.org").unwrap().to_owned(),
+                    TcpStream::connect("127.0.0.1:4190").await.unwrap(),
+                )
+                .await
+                .unwrap(),
+        );
+        SieveConnection {
+            reader: BufReader::new(reader).lines(),
+            writer,
+        }
+    }
+
+    pub async fn authenticate(&mut self, user: &str, pass: &str) {
+        let creds = general_purpose::STANDARD.encode(format!("\0{user}\0{pass}"));
+        self.send(&format!(
+            "AUTHENTICATE PLAIN {{{}+}}\r\n{creds}",
+            creds.len()
+        ))
+        .await;
+        self.assert_read(ResponseType::Ok).await;
+    }
+
+    pub async fn assert_read(&mut self, rt: ResponseType) -> Vec<String> {
+        let lines = self.read().await;
+        let mut buf = Vec::with_capacity(10);
+        rt.serialize(&mut buf);
+        if lines
+            .last()
+            .unwrap()
+            .starts_with(&String::from_utf8(buf).unwrap())
+        {
+            lines
+        } else {
+            panic!("Expected {:?} from server but got: {:?}", rt, lines);
+        }
+    }
+
+    pub async fn read(&mut self) -> Vec<String> {
+        let mut lines = Vec::new();
+        loop {
+            match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+                Ok(Ok(Some(line))) => {
+                    let is_done =
+                        line.starts_with("OK") || line.starts_with("NO") || line.starts_with("BYE");
+                    //println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+
+    pub async fn send_literal(&mut self, text: &str, literal: &str) {
+        self.send(&format!("{}{{{}+}}\r\n{}", text, literal.len(), literal))
+            .await;
+    }
+}

tests/src/utils/smtp.rs

@@ -0,0 +1,192 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::time::Duration;
+
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+
+pub struct SmtpConnection {
+    reader: Lines<BufReader<ReadHalf<TcpStream>>>,
+    writer: WriteHalf<TcpStream>,
+}
+
+impl SmtpConnection {
+    pub async fn ingest_with_code(
+        &mut self,
+        from: &str,
+        recipients: &[&str],
+        message: &str,
+        code: u8,
+    ) -> Vec<String> {
+        self.mail_from(from, 2).await;
+        for recipient in recipients {
+            self.rcpt_to(recipient, 2).await;
+        }
+        self.data(3).await;
+        let result = self.data_bytes(message, recipients.len(), code).await;
+        tokio::time::sleep(Duration::from_millis(500)).await;
+        result
+    }
+
+    pub async fn ingest(&mut self, from: &str, recipients: &[&str], message: &str) {
+        self.ingest_with_code(from, recipients, message, 2).await;
+    }
+
+    pub async fn ingest_chunked(
+        &mut self,
+        from: &str,
+        recipients: &[&str],
+        message: &str,
+        chunk_size: usize,
+    ) {
+        self.mail_from(from, 2).await;
+        for recipient in recipients {
+            self.rcpt_to(recipient, 2).await;
+        }
+        for chunk in message.as_bytes().chunks(chunk_size) {
+            self.bdat(std::str::from_utf8(chunk).unwrap(), 2).await;
+        }
+        self.bdat_last("", recipients.len(), 2).await;
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+
+    pub async fn connect() -> Self {
+        SmtpConnection::connect_port(11200).await
+    }
+
+    pub async fn connect_port(port: u16) -> Self {
+        let (reader, writer) = tokio::io::split(
+            TcpStream::connect(&format!("127.0.0.1:{port}"))
+                .await
+                .unwrap(),
+        );
+        let mut conn = SmtpConnection {
+            reader: BufReader::new(reader).lines(),
+            writer,
+        };
+        conn.read(1, 2).await;
+        conn.lhlo().await;
+        conn
+    }
+
+    pub async fn lhlo(&mut self) -> Vec<String> {
+        self.send("LHLO localhost").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn mail_from(&mut self, sender: &str, code: u8) -> Vec<String> {
+        self.send(&format!("MAIL FROM:<{}>", sender)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn rcpt_to(&mut self, rcpt: &str, code: u8) -> Vec<String> {
+        self.send(&format!("RCPT TO:<{}>", rcpt)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn vrfy(&mut self, rcpt: &str, code: u8) -> Vec<String> {
+        self.send(&format!("VRFY {}", rcpt)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn expn(&mut self, rcpt: &str, code: u8) -> Vec<String> {
+        self.send(&format!("EXPN {}", rcpt)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn data(&mut self, code: u8) -> Vec<String> {
+        self.send("DATA").await;
+        self.read(1, code).await
+    }
+
+    pub async fn data_bytes(
+        &mut self,
+        message: &str,
+        num_responses: usize,
+        code: u8,
+    ) -> Vec<String> {
+        self.send_raw(message).await;
+        self.send_raw("\r\n.\r\n").await;
+        self.read(num_responses, code).await
+    }
+
+    pub async fn bdat(&mut self, chunk: &str, code: u8) -> Vec<String> {
+        self.send_raw(&format!("BDAT {}\r\n{}", chunk.len(), chunk))
+            .await;
+        self.read(1, code).await
+    }
+
+    pub async fn bdat_last(&mut self, chunk: &str, num_responses: usize, code: u8) -> Vec<String> {
+        self.send_raw(&format!("BDAT {} LAST\r\n{}", chunk.len(), chunk))
+            .await;
+        self.read(num_responses, code).await
+    }
+
+    pub async fn rset(&mut self) -> Vec<String> {
+        self.send("RSET").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn noop(&mut self) -> Vec<String> {
+        self.send("NOOP").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn quit(&mut self) -> Vec<String> {
+        self.send("QUIT").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn read(&mut self, mut num_responses: usize, code: u8) -> Vec<String> {
+        let mut lines = Vec::new();
+        loop {
+            match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+                Ok(Ok(Some(line))) => {
+                    let is_done = line.as_bytes()[3] == b' ';
+                    //let c = println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        num_responses -= 1;
+                        if num_responses != 0 {
+                            continue;
+                        }
+
+                        if code != u8::MAX {
+                            for line in &lines {
+                                if line.as_bytes()[0] - b'0' != code {
+                                    panic!("Expected completion code {}, got {:?}.", code, lines);
+                                }
+                            }
+                        }
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+        self.writer.flush().await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+}

tests/src/utils/storage.rs

@@ -0,0 +1,267 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::utils::cleanup::{search_store_destroy, store_assert_is_empty};
+use crate::utils::registry::UnwrapRegistryId;
+use common::Server;
+use registry::schema::structs::{Task, TaskStatus};
+use registry::{
+    schema::{
+        enums::{BlobStoreType, DataStoreType, InMemoryStoreType, SearchStoreType},
+        prelude::Object,
+        structs::{
+            BlobStore, DataStore, ElasticSearchStore, FileSystemStore, FoundationDbStore, HttpAuth,
+            HttpAuthBasic, HttpAuthBearer, InMemoryStore, MeilisearchStore, MySqlStore,
+            PostgreSqlStore, PublicStringOptional, PublicStringValue, RedisStore, RocksDbStore,
+            S3Store, S3StoreCustomRegion, S3StoreRegion, SearchStore, SecretKey, SecretKeyOptional,
+            SecretKeyValue, SqliteStore,
+        },
+    },
+    types::{EnumImpl, duration::Duration},
+};
+use store::write::now;
+use store::{
+    Deserialize, IterateParams, ValueKey,
+    write::{TaskQueueClass, ValueClass},
+};
+use store::{RegistryStore, registry::write::RegistryWrite};
+
+pub trait RegistryEnvStores {
+    fn insert_stores_from_env(&self) -> impl Future<Output = ()>;
+}
+
+impl RegistryEnvStores for RegistryStore {
+    async fn insert_stores_from_env(&self) {
+        let path = self.path().as_os_str().to_str().unwrap();
+        let mut search_store = None;
+        if let Ok(store) = std::env::var("SEARCH_STORE") {
+            let store = SearchStoreType::parse(&store).expect("Invalid store type");
+            search_store = Some(Object::from(build_search_store(store, path).await));
+        }
+        let mut blob_store = None;
+        if let Ok(store) = std::env::var("BLOB_STORE") {
+            let store = BlobStoreType::parse(&store).expect("Invalid store type");
+            blob_store = Some(Object::from(build_blob_store(store, path).await));
+        }
+        let mut in_memory = None;
+        if let Ok(store) = std::env::var("MEMORY_STORE") {
+            let store = InMemoryStoreType::parse(&store).expect("Invalid store type");
+            in_memory = Some(Object::from(build_in_memory_store(store, path).await));
+        }
+
+        for store in [search_store, blob_store, in_memory].into_iter().flatten() {
+            self.write(RegistryWrite::insert(&store))
+                .await
+                .expect("Failed to insert store into registry")
+                .unwrap_id(trc::location!());
+        }
+    }
+}
+
+pub async fn build_data_store(typ: &str, path: &str) -> DataStore {
+    if typ == "MariaDb" {
+        crate::utils::containers::ensure_mariadb().await;
+        return DataStore::MySql(MySqlStore {
+            host: "localhost".into(),
+            port: 3308,
+            auth_username: "root".to_string().into(),
+            auth_secret: SecretKeyOptional::Value(SecretKeyValue {
+                secret: "password".into(),
+            }),
+            database: "stalwart".into(),
+            use_tls: false,
+            allow_invalid_certs: true,
+            ..Default::default()
+        });
+    }
+
+    match DataStoreType::parse(typ).expect("Invalid store type") {
+        DataStoreType::RocksDb => DataStore::RocksDb(RocksDbStore {
+            path: format!("{path}/rocks.db"),
+            ..Default::default()
+        }),
+        DataStoreType::Sqlite => DataStore::Sqlite(SqliteStore {
+            path: format!("{path}/sqlite.db"),
+            ..Default::default()
+        }),
+        DataStoreType::FoundationDb => {
+            crate::utils::containers::ensure_foundationdb().await;
+            DataStore::FoundationDb(FoundationDbStore::default())
+        }
+        DataStoreType::PostgreSql => {
+            crate::utils::containers::ensure_postgres().await;
+            DataStore::PostgreSql(PostgreSqlStore {
+                host: "localhost".into(),
+                port: 5432,
+                auth_username: "stalwart".to_string().into(),
+                auth_secret: SecretKeyOptional::Value(SecretKeyValue {
+                    secret: "stalwart".into(),
+                }),
+                database: "stalwart".into(),
+                use_tls: false,
+                allow_invalid_certs: true,
+                ..Default::default()
+            })
+        }
+        DataStoreType::MySql => {
+            crate::utils::containers::ensure_mysql().await;
+            DataStore::MySql(MySqlStore {
+                host: "localhost".into(),
+                port: 3307,
+                auth_username: "root".to_string().into(),
+                auth_secret: SecretKeyOptional::Value(SecretKeyValue {
+                    secret: "password".into(),
+                }),
+                database: "stalwart".into(),
+                use_tls: false,
+                allow_invalid_certs: true,
+                ..Default::default()
+            })
+        }
+    }
+}
+
+async fn build_blob_store(typ: BlobStoreType, path: &str) -> BlobStore {
+    match typ {
+        BlobStoreType::S3 => {
+            crate::utils::containers::ensure_minio().await;
+            BlobStore::S3(S3Store {
+                access_key: PublicStringOptional::Value(PublicStringValue {
+                    value: "minioadmin".into(),
+                }),
+                bucket: "stalwart".into(),
+                region: S3StoreRegion::Custom(S3StoreCustomRegion {
+                    custom_endpoint: "http://localhost:9000".into(),
+                    custom_region: "eu-central-1".into(),
+                }),
+                secret_key: SecretKeyOptional::Value(SecretKeyValue {
+                    secret: "minioadmin".into(),
+                }),
+                allow_invalid_certs: true,
+                ..Default::default()
+            })
+        }
+        BlobStoreType::FileSystem => BlobStore::FileSystem(FileSystemStore {
+            path: path.to_string(),
+            ..Default::default()
+        }),
+        _ => unreachable!(),
+    }
+}
+
+async fn build_in_memory_store(typ: InMemoryStoreType, _path: &str) -> InMemoryStore {
+    match typ {
+        InMemoryStoreType::Redis => {
+            crate::utils::containers::ensure_redis().await;
+            InMemoryStore::Redis(RedisStore {
+                url: "redis://127.0.0.1".into(),
+                ..Default::default()
+            })
+        }
+        _ => unreachable!(),
+    }
+}
+
+async fn build_search_store(typ: SearchStoreType, _path: &str) -> SearchStore {
+    match typ {
+        SearchStoreType::ElasticSearch => {
+            crate::utils::containers::ensure_opensearch().await;
+            SearchStore::ElasticSearch(ElasticSearchStore {
+                url: "http://localhost:9200".into(),
+                allow_invalid_certs: true,
+                http_auth: HttpAuth::Basic(HttpAuthBasic {
+                    username: "elastic".into(),
+                    secret: SecretKey::Value(SecretKeyValue {
+                        secret: "changeme".into(),
+                    }),
+                }),
+                ..Default::default()
+            })
+        }
+        SearchStoreType::Meilisearch => {
+            crate::utils::containers::ensure_meilisearch().await;
+            SearchStore::Meilisearch(MeilisearchStore {
+                url: "http://localhost:7700".into(),
+                allow_invalid_certs: true,
+                poll_interval: Duration::from_millis(100),
+                http_auth: HttpAuth::Bearer(HttpAuthBearer {
+                    bearer_token: SecretKey::Value(SecretKeyValue {
+                        secret: "stalwart-master-key".into(),
+                    }),
+                }),
+                ..Default::default()
+            })
+        }
+        _ => unreachable!(),
+    }
+}
+
+pub async fn wait_for_tasks(server: &Server, skip_not_due: bool, skip_permanent_failures: bool) {
+    let mut count = 0;
+    loop {
+        let mut has_index_tasks = None;
+        server
+            .core
+            .storage
+            .data
+            .iterate(
+                IterateParams::new(
+                    ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Task { id: 0 })),
+                    ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Task { id: u64::MAX })),
+                )
+                .ascending(),
+                |_, value| {
+                    let task = Task::deserialize(value)?;
+                    if (skip_permanent_failures && matches!(task.status(), TaskStatus::Failed(_)))
+                        || (skip_not_due && task.due_timestamp() > now())
+                    {
+                        Ok(true)
+                    } else {
+                        has_index_tasks = Some(task);
+
+                        Ok(false)
+                    }
+                },
+            )
+            .await
+            .unwrap();
+
+        if let Some(task) = has_index_tasks {
+            count += 1;
+            if count % 10 == 0 {
+                println!("Waiting for pending task {:?}...", task);
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+        } else {
+            break;
+        }
+    }
+}
+
+pub async fn assert_is_empty(server: &Server, include_registry: bool) {
+    // Wait for pending index tasks
+    wait_for_tasks(server, false, false).await;
+
+    // Assert is empty
+    store_assert_is_empty(
+        server.store(),
+        server.core.storage.blob.clone(),
+        include_registry,
+    )
+    .await;
+    search_store_destroy(server.search_store()).await;
+
+    // Clean caches
+    for cache in [
+        &server.inner.cache.events,
+        &server.inner.cache.contacts,
+        &server.inner.cache.files,
+        &server.inner.cache.scheduling,
+    ] {
+        cache.clear();
+    }
+    server.inner.cache.messages.clear();
+}

tests/src/utils/temp_dir.rs

@@ -0,0 +1,37 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub struct TempDir {
+    pub path: std::path::PathBuf,
+    pub delete: bool,
+}
+
+impl TempDir {
+    pub fn new(name: &str, delete_if_exists: bool) -> Self {
+        let mut path = std::env::temp_dir();
+        path.push(name);
+        if delete_if_exists && path.exists() {
+            std::fs::remove_dir_all(&path).unwrap();
+        }
+        std::fs::create_dir_all(&path).unwrap();
+        Self {
+            path,
+            delete: delete_if_exists,
+        }
+    }
+
+    pub fn delete(&self) {
+        std::fs::remove_dir_all(&self.path).unwrap();
+    }
+}
+
+impl Drop for TempDir {
+    fn drop(&mut self) {
+        if self.delete {
+            let _ = std::fs::remove_dir_all(&self.path);
+        }
+    }
+}