Import upstream v0.16.25, stripped

Upstream commit: 3f657330c0f49a015a3a372fb59669b5cccbca6d
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Renamed identifiers: 67 in 19 files
Verification: clean

The same Enterprise footprint as v0.16.24. The build check is clean
apart from the expected errors in the rebuilt-feature tests. A
bug-fix release: DKIM rotation, IMAP failed-login answers, DNSBL
multi-code scoring and negative TTLs, Pyzor on short messages, queue
quotas with an empty match, RocksDB info-log rotation, and
Autodiscover schema handling.

tests/src/automation/dkim.rs

@@ -10,20 +10,37 @@ use common::{config::smtp::auth::Dkim1Signer, network::dns::update::DNS_RECORDS}
 use dns_update::{DnsRecord, NamedDnsRecord};
 use registry::{
     schema::{
-        enums::{DkimRotationStage, DnsRecordType},
-        prelude::ObjectType,
+        enums::{DkimRotationStage, DnsRecordType, IpProtocol, TsigAlgorithm},
+        prelude::{ObjectType, Property},
         structs::{
             CertificateManagement, Dkim1Signature, DkimManagement, DkimManagementProperties,
             DkimSignature, DnsManagement, DnsManagementProperties, DnsServer, DnsServerCloudflare,
-            Domain, SecretKey, SecretKeyValue,
+            DnsServerTsig, Domain, SecretKey, SecretKeyValue, Task, TaskDomainManagement,
+            TaskManager, TaskRetryStrategy, TaskRetryStrategyFixed, TaskStatus,
         },
     },
-    types::duration::Duration,
+    types::{duration::Duration, map::Map},
 };
+use serde_json::json;
 use store::write::now;
 use types::id::Id;
 
+const SHORT_ROTATION_MS: u64 = 6_000;
+const LONG_ROTATION_MS: u64 = 3_600_000;
+
 pub async fn test(test: &TestServer) {
+    fast_retry_tests(test).await;
+    unscheduled_keys_test(test, "dkim-manual.org", |_| DnsManagement::Manual).await;
+    unscheduled_keys_test(test, "dkim-unpublished.org", |dns_server_id| {
+        DnsManagement::Automatic(DnsManagementProperties {
+            dns_server_id,
+            publish_records: Map::new(vec![DnsRecordType::Spf]),
+            ..Default::default()
+        })
+    })
+    .await;
+    automatic_to_manual_dns_test(test).await;
+
     println!("Running DKIM Management tests...");
     let account = test.account("[email protected]");
     DNS_RECORDS.lock().unwrap().clear();
@@ -98,8 +115,8 @@ pub async fn test(test: &TestServer) {
 
     // Make sure the DNS records were created
     let records = DNS_RECORDS.lock().unwrap().clone();
-    assert_key_has_dns_record(&records, &rot1_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&records, &rot1_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_ed25519[0]);
 
     // Expect a rotation to happen and new keys to be created
     let rot2_signatures = account
@@ -111,10 +128,10 @@ pub async fn test(test: &TestServer) {
 
     // Make sure both old and new keys have DNS records
     let records = DNS_RECORDS.lock().unwrap().clone();
-    assert_key_has_dns_record(&records, &rot1_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&records, &rot1_signatures.v1_ed25519[0]);
-    assert_key_has_dns_record(&records, &rot2_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&records, &rot2_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_ed25519[0]);
 
     // Make sure only the new keys are being used for signing
     assert_ne!(
@@ -145,19 +162,19 @@ pub async fn test(test: &TestServer) {
 
     // Make sure the old records were deleted
     let records = DNS_RECORDS.lock().unwrap().clone();
-    assert_key_has_no_dns_record(&records, &rot1_signatures.v1_rsa[0]);
-    assert_key_has_no_dns_record(&records, &rot1_signatures.v1_ed25519[0]);
-    assert_key_has_dns_record(&records, &rot2_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&records, &rot2_signatures.v1_ed25519[0]);
-    assert_key_has_dns_record(&records, &rot3_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&records, &rot3_signatures.v1_ed25519[0]);
+    assert_key_has_no_dns_record(&records, "dkim.org", &rot1_signatures.v1_rsa[0]);
+    assert_key_has_no_dns_record(&records, "dkim.org", &rot1_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot3_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot3_signatures.v1_ed25519[0]);
 
     // Make sure the DNS management task does not republish the retired keys
     let (published, zone_file) = test.published_dkim_records(domain_id).await;
-    assert_key_has_no_dns_record(&published, &rot1_signatures.v1_rsa[0]);
-    assert_key_has_no_dns_record(&published, &rot1_signatures.v1_ed25519[0]);
-    assert_key_has_dns_record(&published, &rot3_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&published, &rot3_signatures.v1_ed25519[0]);
+    assert_key_has_no_dns_record(&published, "dkim.org", &rot1_signatures.v1_rsa[0]);
+    assert_key_has_no_dns_record(&published, "dkim.org", &rot1_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&published, "dkim.org", &rot3_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&published, "dkim.org", &rot3_signatures.v1_ed25519[0]);
     assert_zone_file_omits_key(&zone_file, &rot1_signatures.v1_rsa[0]);
     assert_zone_file_omits_key(&zone_file, &rot1_signatures.v1_ed25519[0]);
 
@@ -192,17 +209,17 @@ pub async fn test(test: &TestServer) {
 
     // Make sure the old records were updated
     let records = DNS_RECORDS.lock().unwrap().clone();
-    assert_key_has_dns_record(&records, &rot4_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&records, &rot4_signatures.v1_ed25519[0]);
-    assert_key_has_no_dns_record(&records, &rot2_signatures.v1_rsa[0]);
-    assert_key_has_no_dns_record(&records, &rot2_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot4_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim.org", &rot4_signatures.v1_ed25519[0]);
+    assert_key_has_no_dns_record(&records, "dkim.org", &rot2_signatures.v1_rsa[0]);
+    assert_key_has_no_dns_record(&records, "dkim.org", &rot2_signatures.v1_ed25519[0]);
 
     // Make sure the DNS management task does not republish the retired keys
     let (published, zone_file) = test.published_dkim_records(domain_id).await;
-    assert_key_has_no_dns_record(&published, &rot2_signatures.v1_rsa[0]);
-    assert_key_has_no_dns_record(&published, &rot2_signatures.v1_ed25519[0]);
-    assert_key_has_dns_record(&published, &rot4_signatures.v1_rsa[0]);
-    assert_key_has_dns_record(&published, &rot4_signatures.v1_ed25519[0]);
+    assert_key_has_no_dns_record(&published, "dkim.org", &rot2_signatures.v1_rsa[0]);
+    assert_key_has_no_dns_record(&published, "dkim.org", &rot2_signatures.v1_ed25519[0]);
+    assert_key_has_dns_record(&published, "dkim.org", &rot4_signatures.v1_rsa[0]);
+    assert_key_has_dns_record(&published, "dkim.org", &rot4_signatures.v1_ed25519[0]);
     assert_zone_file_omits_key(&zone_file, &rot2_signatures.v1_rsa[0]);
     assert_zone_file_omits_key(&zone_file, &rot2_signatures.v1_ed25519[0]);
 
@@ -235,6 +252,470 @@ pub async fn test(test: &TestServer) {
     account.registry_destroy_all(ObjectType::DnsServer).await;
 }
 
+async fn fast_retry_tests(test: &TestServer) {
+    let account = test.account("[email protected]");
+
+    // Retry failed tasks every second
+    account
+        .registry_update_setting(
+            TaskManager {
+                max_attempts: 100,
+                strategy: TaskRetryStrategy::FixedDelay(TaskRetryStrategyFixed {
+                    delay: 1_000u64.into(),
+                }),
+                total_deadline: 86_400_000u64.into(),
+            },
+            &[],
+        )
+        .await;
+    account.reload_settings().await;
+
+    failed_publish_test(test).await;
+    manual_dns_pending_test(test).await;
+
+    account
+        .registry_update_setting(TaskManager::default(), &[])
+        .await;
+    account.reload_settings().await;
+}
+
+async fn failed_publish_test(test: &TestServer) {
+    println!("Running DKIM failed publish tests...");
+    let account = test.account("[email protected]");
+    DNS_RECORDS.lock().unwrap().clear();
+    account.dkim_signatures().await.assert_total(0, 0);
+
+    // Create an in-memory DNS server and a DNS server that refuses connections
+    let dns_server_id = account.create_memory_dns_server().await;
+    let failing_dns_server_id = account.create_failing_dns_server().await;
+
+    // Create a domain whose initial keys rotate shortly
+    let domain_id = account
+        .registry_create_object(Domain {
+            name: "dkim-retry.org".to_string(),
+            certificate_management: CertificateManagement::Manual,
+            dkim_management: dkim_management(SHORT_ROTATION_MS),
+            dns_management: dns_management(dns_server_id),
+            ..Default::default()
+        })
+        .await;
+    let initial = account
+        .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)])
+        .await
+        .assert_total(1, 1);
+    let old_rsa = initial.v1_rsa[0].selector.clone();
+    let old_ed = initial.v1_ed25519[0].selector.clone();
+    test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed])
+        .await;
+
+    // Point the domain at the failing DNS server before the rotation is due, and
+    // make the keys created from now on long-lived
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: dns_management(failing_dns_server_id),
+                Property::DkimManagement: dkim_management(LONG_ROTATION_MS),
+            }),
+        )
+        .await;
+    assert!(
+        initial.v1_rsa[0].next_transition_at.unwrap().timestamp() > now() as i64,
+        "Rotation was due before the DNS server could be replaced: {:#?}",
+        initial
+    );
+
+    // The new keys cannot be published, so they must stay pending while the
+    // old keys remain active and keep signing
+    let failed = account
+        .wait_for_dkim_stages(&[
+            (DkimRotationStage::Pending, 2),
+            (DkimRotationStage::Active, 2),
+        ])
+        .await
+        .assert_total(2, 2)
+        .assert_selector_stage(&old_rsa, DkimRotationStage::Active)
+        .assert_selector_stage(&old_ed, DkimRotationStage::Active);
+    let new_rsa = failed.v1_rsa[0].selector.clone();
+    let new_ed = failed.v1_ed25519[0].selector.clone();
+    let failed = failed
+        .assert_selector_stage(&new_rsa, DkimRotationStage::Pending)
+        .assert_selector_stage(&new_ed, DkimRotationStage::Pending);
+    test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed])
+        .await;
+
+    // Several retries must neither create duplicate keys nor retire the old ones
+    let failure_reason = account.wait_for_dkim_task_attempts(domain_id, 4).await;
+    assert!(
+        failure_reason.contains("Failed to publish DKIM record"),
+        "Unexpected failure reason: {failure_reason}"
+    );
+    let retried = account.dkim_signatures().await;
+    assert_eq!(
+        retried, failed,
+        "DKIM signatures changed while the DNS server was failing"
+    );
+    test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed])
+        .await;
+
+    // Under manual DNS management the pending keys stay pending next to the
+    // active keys, and the task stops retrying
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: DnsManagement::Manual,
+            }),
+        )
+        .await;
+    account.wait_for_no_dkim_tasks(domain_id).await;
+    assert_eq!(
+        account.dkim_signatures().await,
+        failed,
+        "DKIM signatures changed under manual DNS management"
+    );
+    test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed])
+        .await;
+
+    // Once automatic DNS management uses a working server again, the pending
+    // keys are activated and the old keys start retiring
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: dns_management(dns_server_id),
+            }),
+        )
+        .await;
+    let recovered = account
+        .wait_for_dkim_stages(&[
+            (DkimRotationStage::Active, 2),
+            (DkimRotationStage::Retiring, 2),
+        ])
+        .await
+        .assert_total(2, 2)
+        .assert_selector_stage(&new_rsa, DkimRotationStage::Active)
+        .assert_selector_stage(&new_ed, DkimRotationStage::Active)
+        .assert_selector_stage(&old_rsa, DkimRotationStage::Retiring)
+        .assert_selector_stage(&old_ed, DkimRotationStage::Retiring);
+    test.assert_has_signers("dkim-retry.org", &[&new_rsa, &new_ed])
+        .await;
+    let records = DNS_RECORDS.lock().unwrap().clone();
+    assert_key_has_dns_record(&records, "dkim-retry.org", &recovered.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim-retry.org", &recovered.v1_ed25519[0]);
+
+    // Cleanup
+    account.registry_destroy_all(ObjectType::Task).await;
+    account
+        .registry_destroy_all(ObjectType::DkimSignature)
+        .await;
+    account
+        .registry_destroy(ObjectType::Domain, [domain_id])
+        .await
+        .assert_destroyed(&[domain_id]);
+    account.registry_destroy_all(ObjectType::DnsServer).await;
+}
+
+async fn unscheduled_keys_test(
+    test: &TestServer,
+    domain: &str,
+    initial_dns_management: fn(Id) -> DnsManagement,
+) {
+    println!("Running DKIM unscheduled key tests for {domain}...");
+    let account = test.account("[email protected]");
+    DNS_RECORDS.lock().unwrap().clear();
+    account.dkim_signatures().await.assert_total(0, 0);
+    let dns_server_id = account.create_memory_dns_server().await;
+
+    // Keys created while DKIM records are not published automatically are
+    // active, unpublished and have no rotation schedule
+    let domain_id = account
+        .registry_create_object(Domain {
+            name: domain.to_string(),
+            certificate_management: CertificateManagement::Manual,
+            dkim_management: dkim_management(SHORT_ROTATION_MS),
+            dns_management: initial_dns_management(dns_server_id),
+            ..Default::default()
+        })
+        .await;
+    let initial = account
+        .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)])
+        .await
+        .assert_total(1, 1);
+    assert!(
+        initial.keys().all(|key| key.next_transition_at.is_none()),
+        "Unexpected rotation schedule for unpublished keys: {initial:#?}"
+    );
+    let records = DNS_RECORDS.lock().unwrap().clone();
+    assert_key_has_no_dns_record(&records, domain, &initial.v1_rsa[0]);
+    assert_key_has_no_dns_record(&records, domain, &initial.v1_ed25519[0]);
+    let old_rsa = initial.v1_rsa[0].selector.clone();
+    let old_ed = initial.v1_ed25519[0].selector.clone();
+
+    // Publishing DKIM records automatically publishes the keys and schedules
+    // their rotation
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: dns_management(dns_server_id),
+            }),
+        )
+        .await;
+    let scheduled = account
+        .wait_for_dkim("active keys with a rotation schedule", |signatures| {
+            signatures.total() == 2
+                && signatures.stage_count(DkimRotationStage::Active) == 2
+                && signatures
+                    .keys()
+                    .all(|key| key.next_transition_at.is_some())
+        })
+        .await;
+
+    // Make the keys created by the rotation long-lived
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DkimManagement: dkim_management(LONG_ROTATION_MS),
+            }),
+        )
+        .await;
+    assert!(
+        scheduled
+            .keys()
+            .all(|key| key.next_transition_at.unwrap().timestamp() > now() as i64),
+        "Rotation was due before the rotation period could be extended: {scheduled:#?}"
+    );
+    wait_for_dns_records(domain, &[&old_rsa, &old_ed]).await;
+
+    // The keys rotate once the schedule elapses
+    let rotated = account
+        .wait_for_dkim_stages(&[
+            (DkimRotationStage::Active, 2),
+            (DkimRotationStage::Retiring, 2),
+        ])
+        .await
+        .assert_total(2, 2)
+        .assert_selector_stage(&old_rsa, DkimRotationStage::Retiring)
+        .assert_selector_stage(&old_ed, DkimRotationStage::Retiring);
+    test.assert_has_signers(
+        domain,
+        &[&rotated.v1_rsa[0].selector, &rotated.v1_ed25519[0].selector],
+    )
+    .await;
+    let records = DNS_RECORDS.lock().unwrap().clone();
+    assert_key_has_dns_record(&records, domain, &rotated.v1_rsa[0]);
+    assert_key_has_dns_record(&records, domain, &rotated.v1_ed25519[0]);
+
+    // Cleanup
+    account.registry_destroy_all(ObjectType::Task).await;
+    account
+        .registry_destroy_all(ObjectType::DkimSignature)
+        .await;
+    account
+        .registry_destroy(ObjectType::Domain, [domain_id])
+        .await
+        .assert_destroyed(&[domain_id]);
+    account.registry_destroy_all(ObjectType::DnsServer).await;
+}
+
+async fn automatic_to_manual_dns_test(test: &TestServer) {
+    println!("Running DKIM automatic to manual DNS tests...");
+    let account = test.account("[email protected]");
+    DNS_RECORDS.lock().unwrap().clear();
+    account.dkim_signatures().await.assert_total(0, 0);
+    let dns_server_id = account.create_memory_dns_server().await;
+
+    // Create a domain whose initial keys rotate shortly
+    let domain_id = account
+        .registry_create_object(Domain {
+            name: "dkim-mirror.org".to_string(),
+            certificate_management: CertificateManagement::Manual,
+            dkim_management: dkim_management(SHORT_ROTATION_MS),
+            dns_management: dns_management(dns_server_id),
+            ..Default::default()
+        })
+        .await;
+    let initial = account
+        .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)])
+        .await
+        .assert_total(1, 1);
+    let old_rsa = initial.v1_rsa[0].selector.clone();
+    let old_ed = initial.v1_ed25519[0].selector.clone();
+
+    // Switch to manual DNS management before the rotation is due, and make the
+    // keys created from now on long-lived
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: DnsManagement::Manual,
+                Property::DkimManagement: dkim_management(LONG_ROTATION_MS),
+            }),
+        )
+        .await;
+    let due = initial.v1_rsa[0].next_transition_at.unwrap().timestamp();
+    let wait_secs = due - now() as i64;
+    assert!(
+        wait_secs > 0,
+        "Rotation was due before DNS management was switched: {initial:#?}"
+    );
+
+    // Once the rotation is due, the task must neither rotate the keys nor keep
+    // retrying
+    tokio::time::sleep(std::time::Duration::from_secs(wait_secs as u64 + 1)).await;
+    account.wait_for_no_dkim_tasks(domain_id).await;
+    assert_eq!(
+        account.dkim_signatures().await,
+        initial,
+        "DKIM signatures changed under manual DNS management"
+    );
+    test.assert_has_signers("dkim-mirror.org", &[&old_rsa, &old_ed])
+        .await;
+
+    // Switching back to automatic DNS management completes the overdue rotation
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: dns_management(dns_server_id),
+            }),
+        )
+        .await;
+    let rotated = account
+        .wait_for_dkim_stages(&[
+            (DkimRotationStage::Active, 2),
+            (DkimRotationStage::Retiring, 2),
+        ])
+        .await
+        .assert_total(2, 2)
+        .assert_selector_stage(&old_rsa, DkimRotationStage::Retiring)
+        .assert_selector_stage(&old_ed, DkimRotationStage::Retiring);
+    test.assert_has_signers(
+        "dkim-mirror.org",
+        &[&rotated.v1_rsa[0].selector, &rotated.v1_ed25519[0].selector],
+    )
+    .await;
+    let records = DNS_RECORDS.lock().unwrap().clone();
+    assert_key_has_dns_record(&records, "dkim-mirror.org", &rotated.v1_rsa[0]);
+    assert_key_has_dns_record(&records, "dkim-mirror.org", &rotated.v1_ed25519[0]);
+
+    // Cleanup
+    account.registry_destroy_all(ObjectType::Task).await;
+    account
+        .registry_destroy_all(ObjectType::DkimSignature)
+        .await;
+    account
+        .registry_destroy(ObjectType::Domain, [domain_id])
+        .await
+        .assert_destroyed(&[domain_id]);
+    account.registry_destroy_all(ObjectType::DnsServer).await;
+}
+
+async fn manual_dns_pending_test(test: &TestServer) {
+    println!("Running DKIM pending key under manual DNS tests...");
+    let account = test.account("[email protected]");
+    DNS_RECORDS.lock().unwrap().clear();
+    account.dkim_signatures().await.assert_total(0, 0);
+    let failing_dns_server_id = account.create_failing_dns_server().await;
+
+    // Keys created while the DNS server is failing stay pending
+    let domain_id = account
+        .registry_create_object(Domain {
+            name: "dkim-pending.org".to_string(),
+            certificate_management: CertificateManagement::Manual,
+            dkim_management: dkim_management(LONG_ROTATION_MS),
+            dns_management: dns_management(failing_dns_server_id),
+            ..Default::default()
+        })
+        .await;
+    let pending = account
+        .wait_for_dkim_stages(&[(DkimRotationStage::Pending, 2)])
+        .await
+        .assert_total(1, 1);
+    let rsa = pending.v1_rsa[0].selector.clone();
+    let ed = pending.v1_ed25519[0].selector.clone();
+
+    // Switching to manual DNS management activates the pending keys without a
+    // rotation schedule, and the task stops retrying
+    account
+        .registry_update_object(
+            ObjectType::Domain,
+            domain_id,
+            json!({
+                Property::DnsManagement: DnsManagement::Manual,
+            }),
+        )
+        .await;
+    let active = account
+        .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)])
+        .await
+        .assert_total(1, 1)
+        .assert_selector_stage(&rsa, DkimRotationStage::Active)
+        .assert_selector_stage(&ed, DkimRotationStage::Active);
+    assert!(
+        active.keys().all(|key| key.next_transition_at.is_none()),
+        "Unexpected rotation schedule under manual DNS management: {active:#?}"
+    );
+    test.assert_has_signers("dkim-pending.org", &[&rsa, &ed])
+        .await;
+    account.wait_for_no_dkim_tasks(domain_id).await;
+
+    // Cleanup
+    account.registry_destroy_all(ObjectType::Task).await;
+    account
+        .registry_destroy_all(ObjectType::DkimSignature)
+        .await;
+    account
+        .registry_destroy(ObjectType::Domain, [domain_id])
+        .await
+        .assert_destroyed(&[domain_id]);
+    account.registry_destroy_all(ObjectType::DnsServer).await;
+}
+
+fn dns_management(dns_server_id: Id) -> DnsManagement {
+    DnsManagement::Automatic(DnsManagementProperties {
+        dns_server_id,
+        publish_records: Map::new(vec![DnsRecordType::Dkim]),
+        ..Default::default()
+    })
+}
+
+fn dkim_management(rotate_after: u64) -> DkimManagement {
+    DkimManagement::Automatic(DkimManagementProperties {
+        delete_after: Duration::from_millis(LONG_ROTATION_MS),
+        retire_after: Duration::from_millis(LONG_ROTATION_MS),
+        rotate_after: Duration::from_millis(rotate_after),
+        selector_template: "dummy-v{version}-{algorithm}-{epoch}".to_string(),
+        ..Default::default()
+    })
+}
+
+async fn wait_for_dns_records(domain: &str, selectors: &[&str]) {
+    for _ in 0..50 {
+        let records = DNS_RECORDS.lock().unwrap().clone();
+        if selectors
+            .iter()
+            .all(|selector| has_dns_record(&records, domain, selector))
+        {
+            return;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+    }
+    panic!(
+        "DNS records for selectors {selectors:?} were not published: {:#?}",
+        DNS_RECORDS.lock().unwrap()
+    );
+}
+
 #[derive(Debug, PartialEq, Eq, Default)]
 struct DkimSignatures {
     v1_rsa: Vec<Dkim1Signature>,
@@ -265,6 +746,108 @@ impl Account {
         );
     }
 
+    async fn wait_for_dkim_stages(
+        &self,
+        expected: &[(DkimRotationStage, usize)],
+    ) -> DkimSignatures {
+        let expected_total = expected.iter().map(|(_, count)| count).sum::<usize>();
+        self.wait_for_dkim(&format!("stages {expected:?}"), |signatures| {
+            signatures.total() == expected_total
+                && expected
+                    .iter()
+                    .all(|(stage, count)| signatures.stage_count(*stage) == *count)
+        })
+        .await
+    }
+
+    async fn wait_for_dkim(
+        &self,
+        expected: &str,
+        is_expected: impl Fn(&DkimSignatures) -> bool,
+    ) -> DkimSignatures {
+        let mut signatures = self.dkim_signatures().await;
+        for _ in 0..50 {
+            if is_expected(&signatures) {
+                return signatures;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+            signatures = self.dkim_signatures().await;
+        }
+        panic!("DKIM signatures did not reach {expected}: {signatures:#?}");
+    }
+
+    async fn wait_for_no_dkim_tasks(&self, domain_id: Id) {
+        for _ in 0..60 {
+            if self.tasks().await.is_some_and(|tasks| {
+                !tasks.iter().any(|task| {
+                    matches!(&task.task, Task::DkimManagement(task) if task.domain_id == domain_id)
+                })
+            }) {
+                return;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+        }
+        panic!(
+            "DKIM management task did not complete: {:#?}",
+            self.tasks()
+                .await
+                .map(|tasks| tasks.into_iter().map(|task| task.task).collect::<Vec<_>>())
+        );
+    }
+
+    async fn create_failing_dns_server(&self) -> Id {
+        self.registry_create_object(DnsServer::Tsig(DnsServerTsig {
+            host: "127.0.0.1".parse().unwrap(),
+            port: 1,
+            key_name: "stalwart-update-key".to_string(),
+            key: SecretKey::Value(SecretKeyValue {
+                secret: "c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldA==".into(),
+            }),
+            protocol: IpProtocol::Tcp,
+            tsig_algorithm: TsigAlgorithm::HmacSha256,
+            description: "Unreachable DNS server".to_string(),
+            timeout: Duration::from_millis(1_000),
+            ..Default::default()
+        }))
+        .await
+    }
+
+    async fn create_memory_dns_server(&self) -> Id {
+        self.registry_create_object(DnsServer::Cloudflare(DnsServerCloudflare {
+            secret: SecretKey::Value(SecretKeyValue {
+                secret: "[email protected]".into(),
+            }),
+            description: "In-memory DNS server".to_string(),
+            ..Default::default()
+        }))
+        .await
+    }
+
+    async fn wait_for_dkim_task_attempts(&self, domain_id: Id, attempts: u64) -> String {
+        for _ in 0..60 {
+            if let Some(tasks) = self.tasks().await
+                && let Some(failure_reason) = tasks.into_iter().find_map(|task| match task.task {
+                    Task::DkimManagement(TaskDomainManagement {
+                        domain_id: task_domain_id,
+                        status: TaskStatus::Retry(retry),
+                    }) if task_domain_id == domain_id && retry.attempt_number >= attempts => {
+                        Some(retry.failure_reason)
+                    }
+                    _ => None,
+                })
+            {
+                return failure_reason;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+        }
+        panic!(
+            "DKIM management task did not reach {attempts} attempts: {:#?}",
+            self.tasks()
+                .await
+                .map(|tasks| tasks.into_iter().map(|task| task.task).collect::<Vec<_>>())
+        );
+    }
+
     async fn dkim_signatures(&self) -> DkimSignatures {
         let signatures = self.registry_get_all::<DkimSignature>().await;
         let mut v1_rsa = Vec::new();
@@ -286,9 +869,35 @@ impl Account {
 }
 
 impl DkimSignatures {
+    fn keys(&self) -> impl Iterator<Item = &Dkim1Signature> {
+        self.v1_rsa.iter().chain(&self.v1_ed25519)
+    }
+
+    fn total(&self) -> usize {
+        self.v1_rsa.len() + self.v1_ed25519.len()
+    }
+
+    fn stage_count(&self, stage: DkimRotationStage) -> usize {
+        self.v1_rsa.iter().filter(|s| s.stage == stage).count()
+            + self.v1_ed25519.iter().filter(|s| s.stage == stage).count()
+    }
+
+    fn assert_selector_stage(self, selector: &str, stage: DkimRotationStage) -> Self {
+        assert!(
+            self.v1_rsa
+                .iter()
+                .chain(self.v1_ed25519.iter())
+                .any(|s| s.selector == selector && s.stage == stage),
+            "Expected selector {} in stage {:?}: {:#?}",
+            selector,
+            stage,
+            self
+        );
+        self
+    }
+
     fn assert_stage_count(self, stage: DkimRotationStage, count: usize) -> Self {
-        let actual_count = self.v1_rsa.iter().filter(|s| s.stage == stage).count()
-            + self.v1_ed25519.iter().filter(|s| s.stage == stage).count();
+        let actual_count = self.stage_count(stage);
         assert_eq!(
             actual_count, count,
             "Expected {} signatures in stage {:?}, found {}: {:#?}",
@@ -369,21 +978,23 @@ impl TestServer {
     }
 }
 
-fn assert_key_has_dns_record(records: &[NamedDnsRecord], key: &Dkim1Signature) {
-    let expected = format!("{}._domainkey.dkim.org.", key.selector);
-    for record in records {
-        if record.name == expected
-            && let DnsRecord::TXT(txt) = &record.record
-            && ((key.selector.contains("rsa") && txt.starts_with("v=DKIM1; k=rsa; h=sha256; p="))
-                || (key.selector.contains("ed25519")
-                    && txt.starts_with("v=DKIM1; k=ed25519; h=sha256; p=")))
-        {
-            return;
-        }
-    }
-    panic!(
+fn has_dns_record(records: &[NamedDnsRecord], domain: &str, selector: &str) -> bool {
+    let expected = format!("{selector}._domainkey.{domain}.");
+    records.iter().any(|record| {
+        record.name == expected
+            && matches!(&record.record, DnsRecord::TXT(txt)
+                if (selector.contains("rsa") && txt.starts_with("v=DKIM1; k=rsa; h=sha256; p="))
+                    || (selector.contains("ed25519")
+                        && txt.starts_with("v=DKIM1; k=ed25519; h=sha256; p=")))
+    })
+}
+
+fn assert_key_has_dns_record(records: &[NamedDnsRecord], domain: &str, key: &Dkim1Signature) {
+    assert!(
+        has_dns_record(records, domain, &key.selector),
         "No DNS record found for DKIM key with selector {}, records: {:#?}",
-        key.selector, records
+        key.selector,
+        records
     );
 }
 
@@ -396,19 +1007,11 @@ fn assert_zone_file_omits_key(zone_file: &str, key: &Dkim1Signature) {
     );
 }
 
-fn assert_key_has_no_dns_record(records: &[NamedDnsRecord], key: &Dkim1Signature) {
-    let expected = format!("{}._domainkey.dkim.org.", key.selector);
-    for record in records {
-        if record.name == expected
-            && let DnsRecord::TXT(txt) = &record.record
-            && ((key.selector.contains("rsa") && txt.starts_with("v=DKIM1; k=rsa; h=sha256; p="))
-                || (key.selector.contains("ed25519")
-                    && txt.starts_with("v=DKIM1; k=ed25519; h=sha256; p=")))
-        {
-            panic!(
-                "Unexpected DNS record found for DKIM key with selector {}, records: {:#?}",
-                key.selector, records
-            );
-        }
-    }
+fn assert_key_has_no_dns_record(records: &[NamedDnsRecord], domain: &str, key: &Dkim1Signature) {
+    assert!(
+        !has_dns_record(records, domain, &key.selector),
+        "Unexpected DNS record found for DKIM key with selector {}, records: {:#?}",
+        key.selector,
+        records
+    );
 }