diff --git a/.github-upstream/dependabot.yml b/.github-upstream/dependabot.yml index 1d8bc93..c563e91 100644 --- a/.github-upstream/dependabot.yml +++ b/.github-upstream/dependabot.yml @@ -5,11 +5,6 @@ version: 2 updates: - - package-ecosystem: "cargo" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" - # Enable version updates for GitHub Actions - package-ecosystem: "github-actions" # Workflow files stored in the default location of `.github/workflows` diff --git a/.github-upstream/workflows/auto-close-issues.yml b/.github-upstream/workflows/auto-close-issues.yml index 6c34762..9997d4b 100644 --- a/.github-upstream/workflows/auto-close-issues.yml +++ b/.github-upstream/workflows/auto-close-issues.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Close issues from non-allowed authors - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | // Users allowed to open issues directly. All other authors will have diff --git a/.github-upstream/workflows/auto-close-prs.yml b/.github-upstream/workflows/auto-close-prs.yml index fffd048..a5abf34 100644 --- a/.github-upstream/workflows/auto-close-prs.yml +++ b/.github-upstream/workflows/auto-close-prs.yml @@ -18,7 +18,7 @@ jobs: sparse-checkout-cone-mode: false - name: Close PRs from non-allowed authors - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const fs = require('fs'); diff --git a/.github-upstream/workflows/auto-redirect-discussions.yml b/.github-upstream/workflows/auto-redirect-discussions.yml index 3405d94..845abc4 100644 --- a/.github-upstream/workflows/auto-redirect-discussions.yml +++ b/.github-upstream/workflows/auto-redirect-discussions.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Post support portal redirect - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const discussion = context.payload.discussion; diff --git a/.github-upstream/workflows/scorecard.yml b/.github-upstream/workflows/scorecard.yml index 97c4d70..46f46a8 100644 --- a/.github-upstream/workflows/scorecard.yml +++ b/.github-upstream/workflows/scorecard.yml @@ -73,6 +73,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v4.37.4 + uses: github/codeql-action/upload-sarif@v4.38.2 with: sarif_file: results.sarif diff --git a/.github-upstream/workflows/trivy.yml b/.github-upstream/workflows/trivy.yml index 9fb5148..262d6e9 100644 --- a/.github-upstream/workflows/trivy.yml +++ b/.github-upstream/workflows/trivy.yml @@ -36,6 +36,6 @@ jobs: severity: 'CRITICAL,HIGH' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4.37.4 + uses: github/codeql-action/upload-sarif@v4.38.2 with: sarif_file: 'trivy-results.sarif' diff --git a/CHANGELOG.md b/CHANGELOG.md index 300f23a..d999d42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,33 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.16.25] - 2026-10-05 + +If you are upgrading from v0.16.x, replace the binary (or run `docker pull`). If you are upgrading from v0.15.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) for more information on how to upgrade from previous versions. + +## Added + +## Changed + +## Fixed +- JMAP: Creating a `MaskedEmail` with `emailDomain` fails with `forbidden` for every domain when the account has addresses on more than one domain. +- Autodiscover: Requests for a response schema other than Outlook's, such as ActiveSync (`mobilesync`), are answered with the Outlook settings instead of error 601. +- IMAP: + - `LOGIN` and `AUTHENTICATE` with a wrong, expired or unknown app password or API key are answered with an untagged `NO`, so clients keep waiting for the command to complete until the connection times out. + - The failed login that exceeds the maximum number of authentication failures is answered with an untagged `NO` before the connection is closed. +- DKIM: + - A rotation moves the active key to retiring even when its successor fails to publish or propagate, so outgoing mail is sent unsigned until a retry publishes the new key. The DNS write failure is also not logged and the task reports success. + - Keys created while DNS management was manual, or before DKIM was added to the published records, are never rotated after DNS management becomes automatic. Domains already affected start rotating once a `DkimManagement` task is created for them. + - After switching DNS management from automatic to manual, a due rotation activates a new key that was never published in DNS, so signatures fail verification, and retiring the old key is retried forever. +- Spam filter: + - Messages with no text line long enough for a Pyzor digest are checked with the digest of empty input and tagged `PYZOR`. + - DNSBL answers with several return codes, such as a Spamhaus ZEN listing in both SBL and PBL, are scored for only the first code returned. + - DNSBL lookups that return "not listed" are cached for 24 hours regardless of the zone's negative TTL. + - Removing a duplicate training sample of a message reclassified on the same day clears the blob link of the sample that is kept. +- MTA: Queue quotas with an empty `match` expression are never enforced, including the global queue quota created on first start. +- RocksDB: The info log (`LOG`, `LOG.old.*`) grows without limit because log rotation and retention are left at RocksDB defaults. +- WebUI: A blob store read error at startup, such as an S3 authentication failure, stops the web interface from being downloaded. + ## [0.16.24] - 2026-09-27 If you are upgrading from v0.16.x, replace the binary (or run `docker pull`). If you are upgrading from v0.15.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) for more information on how to upgrade from previous versions. diff --git a/Cargo.lock b/Cargo.lock index d7c2a88..9683c4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.48" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb61aea1a7def73ee7c350a184f0e70b32c182344e2e75bf70c9b621b83417fd" +checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1" dependencies = [ "compression-codecs", "compression-core", @@ -1292,7 +1292,7 @@ dependencies = [ [[package]] name = "common" -version = "0.16.24" +version = "0.16.25" dependencies = [ "aes-gcm-siv", "ahash", @@ -1392,9 +1392,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef16c47ba2797aa6a909cc37d39911f3a6743811fe7408ac0b0cc0276b656e9" +checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a" dependencies = [ "compression-core", "flate2", @@ -1477,7 +1477,7 @@ checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "coordinator" -version = "0.16.24" +version = "0.16.25" dependencies = [ "async-nats", "futures", @@ -1889,7 +1889,7 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "dav" -version = "0.16.24" +version = "0.16.25" dependencies = [ "calcard", "chrono", @@ -1912,7 +1912,7 @@ dependencies = [ [[package]] name = "dav-proto" -version = "0.16.24" +version = "0.16.25" dependencies = [ "calcard", "chrono", @@ -2125,7 +2125,7 @@ dependencies = [ [[package]] name = "directory" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "argon2 0.6.0", @@ -2366,7 +2366,7 @@ dependencies = [ [[package]] name = "email" -version = "0.16.24" +version = "0.16.25" dependencies = [ "aes 0.9.3", "aes-gcm 0.11.1", @@ -2406,6 +2406,16 @@ dependencies = [ "log", ] +[[package]] +name = "encodify" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "798c447647dd23f673748f2b868ef309a01dd86aaae999182559d36f06ac82f0" +dependencies = [ + "memchr", + "simdutf8", +] + [[package]] name = "encoding_rs" version = "0.8.42" @@ -2474,7 +2484,7 @@ dependencies = [ [[package]] name = "event_macro" -version = "0.16.24" +version = "0.16.25" dependencies = [ "quote", "syn 3.0.6", @@ -3002,7 +3012,7 @@ dependencies = [ [[package]] name = "groupware" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "calcard", @@ -3289,7 +3299,7 @@ dependencies = [ [[package]] name = "http" -version = "0.16.24" +version = "0.16.25" dependencies = [ "async-stream", "base64 0.23.1", @@ -3385,7 +3395,7 @@ dependencies = [ [[package]] name = "http_proto" -version = "0.16.24" +version = "0.16.25" dependencies = [ "common", "compact_str", @@ -3872,7 +3882,7 @@ checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" [[package]] name = "imap" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "common", @@ -3897,7 +3907,7 @@ dependencies = [ [[package]] name = "imap_proto" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "base64 0.23.1", @@ -3912,7 +3922,7 @@ dependencies = [ [[package]] name = "inbuxa" -version = "0.16.24" +version = "0.16.25" dependencies = [ "common", "coordinator", @@ -3920,7 +3930,7 @@ dependencies = [ "directory", "email", "groupware", - "http 0.16.24", + "http 0.16.25", "http_proto", "imap", "jmap", @@ -4209,7 +4219,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.16.24" +version = "0.16.25" dependencies = [ "async-stream", "base64 0.23.1", @@ -4258,14 +4268,14 @@ dependencies = [ [[package]] name = "jmap-client" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4deab22e057d24e32122f0fc6e2d667a124fdd6a0d8ef3ed4f8a89923c11084f" +checksum = "f5b5bc66252cc8e779ef1238f40ab93971d54ad00b5d7afb5c1d447a9647ed62" dependencies = [ "ahash", "async-stream", - "base64 0.22.1", "chrono", + "encodify", "futures-util", "maybe-async", "parking_lot", @@ -4292,7 +4302,7 @@ dependencies = [ [[package]] name = "jmap_proto" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "calcard", @@ -4502,9 +4512,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" dependencies = [ "spin 0.9.9", ] @@ -4798,7 +4808,7 @@ dependencies = [ [[package]] name = "managesieve" -version = "0.16.24" +version = "0.16.25" dependencies = [ "common", "compact_str", @@ -4933,7 +4943,7 @@ checksum = "c797b9d6bb23aab2fc369c65f871be49214f5c759af65bde26ffaaa2b646b492" [[package]] name = "migration" -version = "0.16.24" +version = "0.16.25" dependencies = [ "common", "email", @@ -5183,7 +5193,7 @@ dependencies = [ [[package]] name = "nlp" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "hashify", @@ -5503,8 +5513,8 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.32.0" -source = "git+https://github.com/stalwartlabs/opentelemetry-rust#80a14a3b6846f62f85506d68d2600c948fccc9d2" +version = "0.33.0" +source = "git+https://github.com/stalwartlabs/opentelemetry-rust#ae66e97b140f70e477ab710686aafce665cc2f8b" dependencies = [ "futures-core", "futures-sink", @@ -5516,8 +5526,8 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.32.0" -source = "git+https://github.com/stalwartlabs/opentelemetry-rust#80a14a3b6846f62f85506d68d2600c948fccc9d2" +version = "0.33.0" +source = "git+https://github.com/stalwartlabs/opentelemetry-rust#ae66e97b140f70e477ab710686aafce665cc2f8b" dependencies = [ "async-trait", "bytes", @@ -5528,8 +5538,8 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.32.0" -source = "git+https://github.com/stalwartlabs/opentelemetry-rust#80a14a3b6846f62f85506d68d2600c948fccc9d2" +version = "0.33.0" +source = "git+https://github.com/stalwartlabs/opentelemetry-rust#ae66e97b140f70e477ab710686aafce665cc2f8b" dependencies = [ "http 1.5.0", "httpdate", @@ -5547,8 +5557,8 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.32.0" -source = "git+https://github.com/stalwartlabs/opentelemetry-rust#80a14a3b6846f62f85506d68d2600c948fccc9d2" +version = "0.33.0" +source = "git+https://github.com/stalwartlabs/opentelemetry-rust#ae66e97b140f70e477ab710686aafce665cc2f8b" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -5559,13 +5569,13 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.1" -source = "git+https://github.com/stalwartlabs/opentelemetry-rust#80a14a3b6846f62f85506d68d2600c948fccc9d2" +version = "0.33.0" +source = "git+https://github.com/stalwartlabs/opentelemetry-rust#ae66e97b140f70e477ab710686aafce665cc2f8b" [[package]] name = "opentelemetry_sdk" -version = "0.32.1" -source = "git+https://github.com/stalwartlabs/opentelemetry-rust#80a14a3b6846f62f85506d68d2600c948fccc9d2" +version = "0.33.0" +source = "git+https://github.com/stalwartlabs/opentelemetry-rust#ae66e97b140f70e477ab710686aafce665cc2f8b" dependencies = [ "futures-channel", "futures-executor", @@ -6015,7 +6025,7 @@ dependencies = [ [[package]] name = "pop3" -version = "0.16.24" +version = "0.16.25" dependencies = [ "common", "directory", @@ -6385,9 +6395,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -6410,9 +6420,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -6835,7 +6845,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "registry" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "hashify", @@ -7392,7 +7402,7 @@ dependencies = [ [[package]] name = "scim" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "base64 0.23.1", @@ -7418,7 +7428,7 @@ dependencies = [ [[package]] name = "scim-proto" -version = "0.16.24" +version = "0.16.25" dependencies = [ "hashify", "serde", @@ -7672,9 +7682,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" dependencies = [ "base64 0.23.1", "bs58", @@ -7693,9 +7703,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -7753,7 +7763,7 @@ dependencies = [ [[package]] name = "services" -version = "0.16.24" +version = "0.16.25" dependencies = [ "aes-gcm 0.11.1", "aho-corasick", @@ -8068,7 +8078,7 @@ checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" [[package]] name = "smtp" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "base64 0.23.1", @@ -8107,9 +8117,9 @@ dependencies = [ [[package]] name = "smtp-proto" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707104487221ff447b5b796b5049e5c09cf52ff9fc1c8abba4695b89ac4b0f37" +checksum = "142a5a642c6bd7ffd7e1b525ad6a6e9921ccef992dba4663974f8519209a00c1" dependencies = [ "memchr", "rkyv", @@ -8159,7 +8169,7 @@ dependencies = [ [[package]] name = "spam-filter" -version = "0.16.24" +version = "0.16.25" dependencies = [ "common", "compact_str", @@ -8279,7 +8289,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "arc-swap", @@ -8539,7 +8549,7 @@ dependencies = [ [[package]] name = "tests" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "aws-lc-rs", @@ -8561,7 +8571,7 @@ dependencies = [ "form_urlencoded", "futures", "groupware", - "http 0.16.24", + "http 0.16.25", "http_proto", "hyper", "hyper-util", @@ -8818,9 +8828,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -9154,7 +9164,7 @@ dependencies = [ [[package]] name = "trc" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "base64 0.23.1", @@ -9263,7 +9273,7 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "types" -version = "0.16.24" +version = "0.16.25" dependencies = [ "blake3", "compact_str", @@ -9432,7 +9442,7 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utils" -version = "0.16.24" +version = "0.16.25" dependencies = [ "ahash", "arcstr", @@ -10117,9 +10127,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yasna" @@ -10144,9 +10154,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", diff --git a/Dockerfile.build b/Dockerfile.build index f1401c9..4e0c3d0 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -4,7 +4,7 @@ # ***************** # Base image for planner & builder # ***************** -FROM --platform=$BUILDPLATFORM rust:slim-trixie AS base +FROM --platform=$BUILDPLATFORM rust:1.98.1-slim-trixie AS base ENV DEBIAN_FRONTEND="noninteractive" \ BINSTALL_DISABLE_TELEMETRY=true \ diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index e305a13..2493472 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "common" -version = "0.16.24" +version = "0.16.25" edition = "2024" build = "build.rs" diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index b0b2b26..f51ebca 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -225,7 +225,7 @@ pub struct Caches { pub dns_ipv6: CacheWithTtl, RecordSet>, pub dns_tlsa: CacheWithTtl, Arc>, pub dns_mta_sts: CacheWithTtl, Arc>, - pub dns_rbl: CacheWithTtl, Option>>, + pub dns_rbl: CacheWithTtl, Option>>, pub negative_cache_ttl: Duration, } diff --git a/crates/common/src/manager/application.rs b/crates/common/src/manager/application.rs index 27a0f03..d2c529e 100644 --- a/crates/common/src/manager/application.rs +++ b/crates/common/src/manager/application.rs @@ -227,10 +227,22 @@ impl WebApplicationManager { let cached = if force_refresh { None } else { - server + match server .blob_store() .get_blob(self.blob_key.as_slice(), 0..usize::MAX) - .await? + .await + { + Ok(cached) => cached, + Err(err) => { + trc::event!( + Resource(trc::ResourceEvent::Error), + Reason = err, + Url = self.url.clone(), + Details = "Failed to read cached application bundle, downloading it again" + ); + None + } + } }; let is_cached = cached.is_some(); let bundle = match cached { diff --git a/crates/common/src/network/autoconfig/autodiscover.rs b/crates/common/src/network/autoconfig/autodiscover.rs index eb17351..4ed3819 100644 --- a/crates/common/src/network/autoconfig/autodiscover.rs +++ b/crates/common/src/network/autoconfig/autodiscover.rs @@ -11,7 +11,7 @@ use quick_xml::Reader; use quick_xml::XmlVersion; use quick_xml::events::Event; use registry::schema::{enums::ServiceProtocol, structs::Service}; -use std::fmt::Write; +use std::{borrow::Cow, fmt::Write}; use utils::map::vec_map::VecMap; impl Server { @@ -20,32 +20,78 @@ impl Server { body: Option>, ) -> trc::Result>> { // Obtain parameters - let emailaddress = parse_autodiscover_request(body.as_deref().unwrap_or_default()) - .map_err(|err| { + let request = + parse_autodiscover_request(body.as_deref().unwrap_or_default()).map_err(|err| { trc::ResourceEvent::BadParameters .into_err() .details("Failed to parse autodiscover request") .ctx(trc::Key::Reason, err) })?; // inbuxa: legacy-protocols LP-7, LP-14a - let legacy_off = match emailaddress.rsplit_once('@') { + let legacy_off = match request.email.rsplit_once('@') { Some((_, domain)) => self.legacy_off_for(domain).await?, None => self.legacy_off_for("").await?, }; - Ok(Resource::new( - "application/xml; charset=utf-8", - build_autodiscover_response( - &emailaddress, + let response = match request.response_schema { + ResponseSchema::Outlook => build_autodiscover_response( + &request.email, &self.core.network.server_name, &self.core.network.info.services, |protocol| legacy_off.service(protocol), ) .into_bytes(), - )) + ResponseSchema::Unsupported => PROVIDER_NOT_AVAILABLE_RESPONSE.as_bytes().to_vec(), + }; + + Ok(Resource::new("application/xml; charset=utf-8", response)) } } +const OUTLOOK_RESPONSE_SCHEMA: &str = + "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"; + +const PROVIDER_NOT_AVAILABLE_RESPONSE: &str = concat!( + "\n", + "\n", + "\t\n", + "\t\t\n", + "\t\t\t601\n", + "\t\t\tProvider is not available\n", + "\t\t\t\n", + "\t\t\n", + "\t\n", + "\n", +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ResponseSchema { + Outlook, + Unsupported, +} + +impl ResponseSchema { + fn parse(value: &str) -> Self { + if value.trim().eq_ignore_ascii_case(OUTLOOK_RESPONSE_SCHEMA) { + ResponseSchema::Outlook + } else { + ResponseSchema::Unsupported + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct AutodiscoverRequest { + email: String, + response_schema: ResponseSchema, +} + +#[derive(Clone, Copy)] +enum RequestField { + EmailAddress, + ResponseSchema, +} + fn build_autodiscover_response( emailaddress: &str, default_host: &str, @@ -124,7 +170,7 @@ fn build_autodiscover_response( config } -fn parse_autodiscover_request(bytes: &[u8]) -> Result { +fn parse_autodiscover_request(bytes: &[u8]) -> Result { if bytes.is_empty() { return Err("Empty request body".to_string()); } @@ -132,8 +178,9 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { let mut reader = Reader::from_reader(bytes); reader.config_mut().trim_text(true); let mut buf = Vec::with_capacity(128); + let mut value_buf = Vec::with_capacity(128); - 'outer: for tag_name in ["Autodiscover", "Request", "EMailAddress"] { + 'outer: for tag_name in ["Autodiscover", "Request"] { loop { match reader.read_event_into(&mut buf) { Ok(Event::Start(e)) => { @@ -143,30 +190,6 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { .eq_ignore_ascii_case(found_tag_name.as_ref()) { continue 'outer; - } else if tag_name == "EMailAddress" { - // Skip unsupported tags under Request, such as AcceptableResponseSchema - let mut tag_count = 0; - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::End(_)) => { - if tag_count == 0 { - break; - } else { - tag_count -= 1; - } - } - Ok(Event::Start(_)) => { - tag_count += 1; - } - Ok(Event::Eof) => { - return Err(format!( - "Expected value, found unexpected EOF at position {}.", - reader.buffer_position() - )); - } - _ => (), - } - } } else { return Err(format!( "Expected tag {}, found unexpected tag {} at position {}.", @@ -195,36 +218,170 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { } } - if let Ok(Event::Text(text)) = reader.read_event_into(&mut buf) - && let Ok(text) = text.xml_content(XmlVersion::Implicit1_0) - && text.contains('@') - { - return Ok(text.trim().to_lowercase()); + let mut email = None; + let mut response_schema = ResponseSchema::Outlook; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) => { + let local_name = e.local_name(); + let field = hashify::tiny_map_ignore_case!(local_name.as_ref(), + b"EMailAddress" => RequestField::EmailAddress, + b"AcceptableResponseSchema" => RequestField::ResponseSchema, + ); + + let value = match reader.read_event_into(&mut value_buf) { + Ok(Event::End(_)) => None, + Ok(event) => { + let value = match event { + Event::Text(text) => text + .xml_content(XmlVersion::Implicit1_0) + .ok() + .map(Cow::into_owned), + _ => None, + }; + reader + .read_to_end_into(e.name(), &mut value_buf) + .map_err(|err| { + format!("Error at position {}: {:?}", reader.buffer_position(), err) + })?; + value + } + Err(err) => { + return Err(format!( + "Error at position {}: {:?}", + reader.buffer_position(), + err + )); + } + }; + + match (field, value) { + (Some(RequestField::EmailAddress), Some(value)) => { + email = Some(value); + } + (Some(RequestField::ResponseSchema), Some(value)) => { + response_schema = ResponseSchema::parse(&value); + } + _ => (), + } + } + Ok(Event::End(_) | Event::Eof) => break, + Ok(_) => (), + Err(e) => { + return Err(format!( + "Error at position {}: {:?}", + reader.buffer_position(), + e + )); + } + } } - Err(format!( - "Expected email address, found unexpected value at position {}.", - reader.buffer_position() - )) + match email { + Some(email) if email.contains('@') => Ok(AutodiscoverRequest { + email: email.trim().to_lowercase(), + response_schema, + }), + _ => Err(format!( + "Expected email address, found unexpected value at position {}.", + reader.buffer_position() + )), + } } #[cfg(test)] mod tests { + use super::{AutodiscoverRequest, ResponseSchema, parse_autodiscover_request}; #[test] fn parse_autodiscover() { - let r = r#" + const OUTLOOK: &str = + "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"; + const MOBILESYNC: &str = + "http://schemas.microsoft.com/exchange/autodiscover/mobilesync/responseschema/2006"; + + for (request, expected) in [ + ( + format!( + r#" - email@example.com - http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a + Email@Example.com + {OUTLOOK} - "#; + "# + ), + ResponseSchema::Outlook, + ), + ( + format!( + r#" + + {OUTLOOK} + email@example.com + + "# + ), + ResponseSchema::Outlook, + ), + ( + r#" + + email@example.com + + "# + .to_string(), + ResponseSchema::Outlook, + ), + ( + format!( + r#" + + + email@example.com + {MOBILESYNC} + + "# + ), + ResponseSchema::Unsupported, + ), + ( + format!( + r#" + + /o=Example/ou=Users/cn=email + value + {MOBILESYNC} + email@example.com + + "# + ), + ResponseSchema::Unsupported, + ), + ] { + assert_eq!( + parse_autodiscover_request(request.as_bytes()).expect("valid request"), + AutodiscoverRequest { + email: "email@example.com".to_string(), + response_schema: expected, + }, + "{request}" + ); + } - assert_eq!( - super::parse_autodiscover_request(r.as_bytes()).unwrap(), - "email@example.com" - ); + for request in [ + "", + "", + "no-domain", + "email@example.com", + "email@example.com", + ] { + assert!( + parse_autodiscover_request(request.as_bytes()).is_err(), + "{request}" + ); + } } #[test] diff --git a/crates/coordinator/Cargo.toml b/crates/coordinator/Cargo.toml index 0db24a7..6751dfa 100644 --- a/crates/coordinator/Cargo.toml +++ b/crates/coordinator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "coordinator" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/dav-proto/Cargo.toml b/crates/dav-proto/Cargo.toml index b4d63b8..550b2e1 100644 --- a/crates/dav-proto/Cargo.toml +++ b/crates/dav-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dav-proto" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/dav/Cargo.toml b/crates/dav/Cargo.toml index 58a3873..e59c6b7 100644 --- a/crates/dav/Cargo.toml +++ b/crates/dav/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dav" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 063479d..d123cc5 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "directory" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml index 40ce0cc..a595d0d 100644 --- a/crates/email/Cargo.toml +++ b/crates/email/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "email" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 7538ee4..2d2bc6c 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -20,7 +20,7 @@ use groupware::{ scheduling::{ItipError, ItipMessages}, }; use mail_parser::{ - DateTime, Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, + Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, parsers::fields::thread::thread_name, }; use registry::{ @@ -924,11 +924,7 @@ impl EmailIngest for Server { span_id: u64, ) { if let Some(config) = &self.core.spam.classifier { - let mut dt = DateTime::from_timestamp(now() as i64); - dt.hour = 0; - dt.minute = 0; - dt.second = 0; - let until = dt.to_timestamp() as u64 + config.hold_samples_for; + let until = now() + config.hold_samples_for; let sample = SpamTrainingSample { account_id: Some(Id::from(account_id)), diff --git a/crates/groupware/Cargo.toml b/crates/groupware/Cargo.toml index a1daf0e..3540044 100644 --- a/crates/groupware/Cargo.toml +++ b/crates/groupware/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "groupware" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/http-proto/Cargo.toml b/crates/http-proto/Cargo.toml index 3110362..5de066d 100644 --- a/crates/http-proto/Cargo.toml +++ b/crates/http-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "http_proto" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index 99578d2..66b2384 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "http" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index f06702c..9331a57 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap_proto" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/imap-proto/src/protocol/mod.rs b/crates/imap-proto/src/protocol/mod.rs index 3c511bb..d3f4dab 100644 --- a/crates/imap-proto/src/protocol/mod.rs +++ b/crates/imap-proto/src/protocol/mod.rs @@ -677,7 +677,13 @@ pub trait SerializeResponse { impl SerializeResponse for trc::Error { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity(128); - if let Some(tag) = self.value_as_str(trc::Key::Id) { + if let Some(tag) = self + .keys() + .iter() + .rev() + .find_map(|(key, value)| (*key == trc::Key::Id).then_some(value)) + .and_then(|value| value.as_str()) + { buf.extend_from_slice(tag.as_bytes()); } else { buf.push(b'*'); @@ -813,9 +819,51 @@ impl Display for Command { #[cfg(test)] mod tests { use crate::parser::parse_sequence_set; - use crate::protocol::ObjectId; + use crate::protocol::{ObjectId, SerializeResponse}; use types::id::Id; + #[test] + fn serialize_error_uses_command_tag() { + for (error, expected) in [ + ( + trc::AuthEvent::Failed.into_err().id("a1"), + "a1 NO [AUTHENTICATIONFAILED] ", + ), + ( + trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::Id, 7u32) + .id("a1"), + "a1 NO [AUTHENTICATIONFAILED] ", + ), + ( + trc::AuthEvent::Error + .into_err() + .ctx(trc::Key::Id, "12") + .id("a2"), + "a2 NO [AUTHENTICATIONFAILED] ", + ), + ( + trc::AuthEvent::TooManyAttempts + .into_err() + .caused_by(trc::AuthEvent::Failed.into_err().ctx(trc::Key::Id, 7u32)) + .id("a3"), + "a3 NO [AUTHENTICATIONFAILED] ", + ), + ( + trc::AuthEvent::Failed.into_err().ctx(trc::Key::Id, 7u32), + "* NO [AUTHENTICATIONFAILED] ", + ), + ] { + let response = error.serialize(); + assert!( + response.starts_with(expected.as_bytes()), + "{:?} does not start with {expected:?}", + String::from_utf8_lossy(&response) + ); + } + } + #[test] fn serialize_objectid_compound() { // Empty compound diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index cc638c3..4580e9e 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/imap/src/op/authenticate.rs b/crates/imap/src/op/authenticate.rs index 5391920..58c42b3 100644 --- a/crates/imap/src/op/authenticate.rs +++ b/crates/imap/src/op/authenticate.rs @@ -92,7 +92,10 @@ impl Session { auth_failures: auth_failures + 1, }; } else { - return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); + return trc::AuthEvent::TooManyAttempts + .into_err() + .caused_by(err) + .id(tag.clone()); } } diff --git a/crates/jmap-proto/Cargo.toml b/crates/jmap-proto/Cargo.toml index 9ad5ad7..bdba535 100644 --- a/crates/jmap-proto/Cargo.toml +++ b/crates/jmap-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap_proto" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 3198a40..3ab7325 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/jmap/src/registry/mapping/bootstrap.rs b/crates/jmap/src/registry/mapping/bootstrap.rs index 3960da9..c47c7e0 100644 --- a/crates/jmap/src/registry/mapping/bootstrap.rs +++ b/crates/jmap/src/registry/mapping/bootstrap.rs @@ -80,9 +80,9 @@ pub(crate) async fn bootstrap_set( mut set: RegistrySetResponse<'_>, ) -> trc::Result> { if !set.server.registry().is_bootstrap_mode() { - set.fail_all_create("This operation is only allowed bootstrap mode"); - set.fail_all_update("This operation is only allowed bootstrap mode"); - set.fail_all_destroy("This operation is only allowed bootstrap mode"); + set.fail_all_create("This operation is only allowed in bootstrap mode"); + set.fail_all_update("This operation is only allowed in bootstrap mode"); + set.fail_all_destroy("This operation is only allowed in bootstrap mode"); return Ok(set); } diff --git a/crates/jmap/src/registry/mapping/domain.rs b/crates/jmap/src/registry/mapping/domain.rs index e1e370e..1d70388 100644 --- a/crates/jmap/src/registry/mapping/domain.rs +++ b/crates/jmap/src/registry/mapping/domain.rs @@ -102,6 +102,10 @@ pub(crate) async fn validate_domain( let will_trigger_dkim = matches!(domain.dkim_management, DkimManagement::Automatic(_)) && old_domain .is_none_or(|old| !matches!(old.dkim_management, DkimManagement::Automatic(_))); + let will_schedule_dkim = !will_trigger_dkim + && matches!(domain.dkim_management, DkimManagement::Automatic(_)) + && publishes_dkim(domain) + && old_domain.is_some_and(|old| !publishes_dkim(old)); let will_trigger_acme = if let DnsManagement::Automatic(details) = &domain.dns_management && old_domain.is_none_or(|old| !matches!(old.dns_management, DnsManagement::Automatic(_))) { @@ -125,11 +129,19 @@ pub(crate) async fn validate_domain( })); on_success_renew_certificate } else { + if will_schedule_dkim { + tasks.push(Task::DnsManagement(TaskDnsManagement { + domain_id: Id::default(), + update_records: Map::new(vec![DnsRecordType::Dkim]), + on_success_renew_certificate: false, + status: TaskStatus::now(), + })); + } false }; // Schedule DKIM key rotation task - if will_trigger_dkim { + if will_trigger_dkim || will_schedule_dkim { tasks.push(Task::DkimManagement(TaskDomainManagement { domain_id: Id::default(), status: TaskStatus::now(), @@ -176,6 +188,13 @@ pub(crate) async fn validate_domain( Ok(Ok(response)) } +fn publishes_dkim(domain: &Domain) -> bool { + matches!( + &domain.dns_management, + DnsManagement::Automatic(details) if details.publish_records.contains(&DnsRecordType::Dkim) + ) +} + pub(crate) async fn validate_dns_server( set: &RegistrySetResponse<'_>, dns: &mut DnsServer, diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index e88f62f..64ebcdd 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ keywords = ["imap", "jmap", "smtp", "email", "mail", "webdav", "server"] categories = ["email"] # Upstream offers AGPL-3.0-only OR LicenseRef-SEL; inbuxa takes the AGPL only. license = "AGPL-3.0-only" -version = "0.16.24" +version = "0.16.25" edition = "2024" [[bin]] diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 7be7f60..9b02fd7 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "managesieve" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/migration/Cargo.toml b/crates/migration/Cargo.toml index ddb8652..f711de3 100644 --- a/crates/migration/Cargo.toml +++ b/crates/migration/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "migration" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index 3a23975..b8c9541 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nlp" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/pop3/Cargo.toml b/crates/pop3/Cargo.toml index de7c540..211b87c 100644 --- a/crates/pop3/Cargo.toml +++ b/crates/pop3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pop3" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/registry/Cargo.toml b/crates/registry/Cargo.toml index f918a37..263c2a1 100644 --- a/crates/registry/Cargo.toml +++ b/crates/registry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "registry" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/scim-proto/Cargo.toml b/crates/scim-proto/Cargo.toml index b66bf6e..bd82239 100644 --- a/crates/scim-proto/Cargo.toml +++ b/crates/scim-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scim-proto" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/scim/Cargo.toml b/crates/scim/Cargo.toml index 9fa6469..05055ee 100644 --- a/crates/scim/Cargo.toml +++ b/crates/scim/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scim" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index b434f51..22fe66a 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "services" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/services/src/task_manager/dkim.rs b/crates/services/src/task_manager/dkim.rs index 59ab0f1..341831d 100644 --- a/crates/services/src/task_manager/dkim.rs +++ b/crates/services/src/task_manager/dkim.rs @@ -68,12 +68,14 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R "Domain is not set to automatic DKIM management".to_string(), )); }; - let mut create_signatures = dkim.algorithms.into_inner(); - if create_signatures.is_empty() { + let configured = dkim.algorithms.into_inner(); + if configured.is_empty() { return Ok(TaskResult::permanent( "No DKIM algorithms configured for domain".to_string(), )); } + let mut create_signatures = configured.clone(); + let mut active_types: Vec = Vec::with_capacity(configured.len()); let dns_updater = match domain.dns_management { DnsManagement::Automatic(props) if props.publish_records.contains(&DnsRecordType::Dkim) => { @@ -95,6 +97,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R let mut retire_signatures = Vec::new(); let mut retiring_signatures = Vec::new(); let mut delete_signatures = Vec::new(); + let mut schedule_signatures = Vec::new(); let mut next_transition = None; let signature_ids = server @@ -123,16 +126,35 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R create_signatures.retain(|algo| algo != &key_algo); publish_signatures.push(key) } - DkimRotationStage::Active => retiring_signatures.push(key), + DkimRotationStage::Active if dns_updater.is_some() => retiring_signatures.push(key), + DkimRotationStage::Active => { + create_signatures.retain(|algo| algo != &key_algo); + active_types.push(key_algo); + } DkimRotationStage::Retiring => retire_signatures.push(key), DkimRotationStage::Retired => delete_signatures.push(key), } } else { - if key.object.is_active() { - create_signatures.retain(|algo| algo != &key_algo); + let transition = key.object.next_transition(); + match key.object.stage() { + DkimRotationStage::Active => { + create_signatures.retain(|algo| algo != &key_algo); + active_types.push(key_algo); + if transition.is_none() && dns_updater.is_some() { + schedule_signatures.push(key); + } + } + DkimRotationStage::Pending => { + create_signatures.retain(|algo| algo != &key_algo); + if transition.is_none() || dns_updater.is_none() { + publish_signatures.push(key); + continue; + } + } + DkimRotationStage::Retiring | DkimRotationStage::Retired => {} } - if let Some(transition) = key.object.next_transition() + if let Some(transition) = transition && next_transition.is_none_or(|next| transition < next) { next_transition = Some(transition); @@ -142,6 +164,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R let now = now(); let mut do_refresh = false; + let mut temporary_errors = String::new(); for algorithm in create_signatures { #[cfg(feature = "test_mode")] @@ -221,7 +244,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R )); }; let propagation_target = txt_value.clone(); - let published = updater + let published = match updater .set_rrset( origin, &record.name, @@ -229,12 +252,43 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R vec![record.record.clone()], ) .await - .is_ok(); - let signature_transition = if published - && updater - .wait_for_txt_propagation(&record.name, origin, &propagation_target) - .await { + Ok(_) => { + let propagated = updater + .wait_for_txt_propagation(&record.name, origin, &propagation_target) + .await; + if !propagated { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "DKIM record {} did not propagate, will retry.", + record.name + ); + } + propagated + } + Err(err) => { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "Failed to publish DKIM record {}: {err}.", + record.name + ); + trc::event!( + Dns(DnsEvent::RecordCreationFailed), + Hostname = record.name.clone(), + Details = origin.clone(), + Type = "TXT", + Reason = err, + ); + false + } + }; + let signature_transition = if published { trc::event!( Dkim(DkimEvent::SignaturePublished), Id = selector.clone(), @@ -246,7 +300,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } else { // Something went wrong, reschedule. signature.set_stage(DkimRotationStage::Pending); - UTCDateTime::from_timestamp((now + 60) as i64) // Retry after 1 minute + UTCDateTime::from_timestamp(now as i64) }; if next_transition.is_none_or(|next| signature_transition < next) { @@ -257,12 +311,16 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } // Write key + let is_active = signature.is_active(); match server .registry() .write(RegistryWrite::insert(&signature.into())) .await? { RegistryWriteResult::Success(_) => { + if is_active { + active_types.push(algorithm); + } trc::event!( Dkim(DkimEvent::SignatureCreated), Id = selector, @@ -277,11 +335,35 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } } + for signature in schedule_signatures { + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); + let signature_transition = + UTCDateTime::from_timestamp((now + dkim.rotate_after.as_secs()) as i64); + + if next_transition.is_none_or(|next| signature_transition < next) { + next_transition = Some(signature_transition); + } + + let mut new_signature = signature.object.clone(); + new_signature.set_next_transition(signature_transition); + + if let SignatureUpdate::Failed(task_result) = update_signature( + server, + signature, + new_signature, + &record, + &mut temporary_errors, + ) + .await? + { + return Ok(task_result); + } + } + // Publish signatures - let mut temporary_errors = String::new(); - for signature in publish_signatures { - let record = generate_dkim_dns_record(&signature.object, &domain.name).await?; - if let Some((updater, origin)) = &dns_updater { + if let Some((updater, origin)) = &dns_updater { + for signature in publish_signatures { + let record = generate_dkim_dns_record(&signature.object, &domain.name).await?; let dns_update::DnsRecord::TXT(txt_value) = &record.record else { return Ok(TaskResult::permanent( "DKIM record must be a TXT record".to_string(), @@ -311,6 +393,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R next_transition = Some(signature_transition); } + let signature_type = signature.object.object_type(); let mut new_signature = signature.object.clone(); new_signature.set_next_transition(signature_transition); @@ -323,7 +406,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ); // Write key - if let Some(task_result) = update_signature( + match update_signature( server, signature, new_signature, @@ -332,7 +415,9 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ) .await? { - return Ok(task_result); + SignatureUpdate::Written => active_types.push(signature_type), + SignatureUpdate::Conflict => {} + SignatureUpdate::Failed(task_result) => return Ok(task_result), } do_refresh = true; } @@ -357,20 +442,52 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ); } } - } else { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); + } + } else { + for signature in publish_signatures { + let signature_type = signature.object.object_type(); + if active_types.contains(&signature_type) || !configured.contains(&signature_type) { + continue; } - let _ = write!( + + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); + let mut new_signature = signature.object.clone(); + new_signature.set_stage(DkimRotationStage::Active); + match &mut new_signature { + DkimSignature::Dkim1Ed25519Sha256(sign) | DkimSignature::Dkim1RsaSha256(sign) => { + sign.next_transition_at = None + } + DkimSignature::Dkim2Ed25519Sha256(sign) | DkimSignature::Dkim2RsaSha256(sign) => { + sign.next_transition_at = None + } + } + + match update_signature( + server, + signature, + new_signature, + &record, &mut temporary_errors, - "No DNS server configured, cannot publish DKIM record {}.", - record.name - ); + ) + .await? + { + SignatureUpdate::Written => { + active_types.push(signature_type); + do_refresh = true; + } + SignatureUpdate::Conflict => active_types.push(signature_type), + SignatureUpdate::Failed(task_result) => return Ok(task_result), + } } } // Retiring signatures for signature in retiring_signatures { + let signature_type = signature.object.object_type(); + if configured.contains(&signature_type) && !active_types.contains(&signature_type) { + continue; + } + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); let signature_transition = UTCDateTime::from_timestamp((now + dkim.retire_after.as_secs()) as i64); @@ -391,7 +508,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ); // Write key - if let Some(task_result) = update_signature( + if let SignatureUpdate::Failed(task_result) = update_signature( server, signature, new_signature, @@ -406,9 +523,9 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } // Retire signatures - for signature in retire_signatures { - let record = generate_dkim_dns_record_name(&signature.object, &domain.name); - if let Some((updater, origin)) = &dns_updater { + if let Some((updater, origin)) = &dns_updater { + for signature in retire_signatures { + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); match updater .set_rrset(origin, &record, dns_update::DnsRecordType::TXT, Vec::new()) .await @@ -433,7 +550,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ); // Write key - if let Some(task_result) = update_signature( + if let SignatureUpdate::Failed(task_result) = update_signature( server, signature, new_signature, @@ -458,15 +575,6 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ); } } - } else { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( - &mut temporary_errors, - "No DNS server configured, cannot retire DKIM record {}.", - record - ); } } @@ -556,13 +664,19 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } } +enum SignatureUpdate { + Written, + Conflict, + Failed(TaskResult), +} + async fn update_signature( server: &Server, signature: RegistryObject, new_signature: DkimSignature, name: &str, temporary_errors: &mut String, -) -> trc::Result> { +) -> trc::Result { match server .registry() .write(RegistryWrite::update( @@ -575,8 +689,8 @@ async fn update_signature( )) .await { - Ok(RegistryWriteResult::Success(_)) => Ok(None), - Ok(err) => Ok(Some(TaskResult::permanent(format!( + Ok(RegistryWriteResult::Success(_)) => Ok(SignatureUpdate::Written), + Ok(err) => Ok(SignatureUpdate::Failed(TaskResult::permanent(format!( "Failed to write DKIM signature for record {name}: {err}" )))), Err(err) => { @@ -588,7 +702,7 @@ async fn update_signature( temporary_errors, "Failed to write DKIM signature for record {name} due to concurrent modification, will retry.", ); - Ok(None) + Ok(SignatureUpdate::Conflict) } else { Err(err) } diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index f2d40ec..8a68f5e 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -6,7 +6,7 @@ homepage = "https://inbuxa.org" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs index 7305899..699f103 100644 --- a/crates/smtp/src/queue/quota.rs +++ b/crates/smtp/src/queue/quota.rs @@ -127,8 +127,8 @@ impl HasQueueQuota for Server { refs: &mut Vec, session_id: u64, ) -> bool { - if !quota.expr.is_empty() - && self + if quota.expr.is_empty() + || self .eval_if("a.expr, envelope, session_id) .await .unwrap_or(false) diff --git a/crates/spam-filter/Cargo.toml b/crates/spam-filter/Cargo.toml index b842c2e..67e5d81 100644 --- a/crates/spam-filter/Cargo.toml +++ b/crates/spam-filter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spam-filter" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index dca505a..2bb5e44 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -37,7 +37,7 @@ use std::{ hash::{Hash, RandomState}, sync::Arc, }; -use store::ahash::AHashSet; +use store::ahash::AHashMap; use store::rand::seq::SliceRandom; use store::write::{BlobLink, RegistryClass, now}; use store::{ @@ -92,7 +92,14 @@ struct TrainingTask { sample: TrainingSample, is_spam: bool, is_replay: bool, - remove: Option, + remove: SampleRemoval, +} + +#[derive(Debug, Clone, Copy)] +enum SampleRemoval { + Keep, + Item, + ItemAndLink { until: u64 }, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] @@ -199,7 +206,7 @@ impl SpamClassifier for Server { object_id, item_id: u64::MAX, })); - let mut seen_samples = AHashSet::new(); + let mut seen_samples = AHashMap::new(); let mut spam_count = 0; let mut ham_count = 0; self.store() @@ -220,43 +227,57 @@ impl SpamClassifier for Server { .unwrap_or(u32::MAX), }; - if seen_samples.insert(sample.clone()) { - // Add to reservoir - if !do_remove { - trainer.reservoir.update_reservoir( - &sample, + match seen_samples.entry(sample.clone()) { + Entry::Vacant(entry) => { + entry.insert((!do_remove).then_some(until)); + + // Add to reservoir + if !do_remove { + trainer.reservoir.update_reservoir( + &sample, + is_spam, + config.reservoir_capacity, + ); + } else { + trainer.reservoir.update_counts(is_spam); + } + + samples.push(TrainingTask { + id, + sample, is_spam, - config.reservoir_capacity, - ); - } else { - trainer.reservoir.update_counts(is_spam); + is_replay: false, + remove: if do_remove { + SampleRemoval::ItemAndLink { until } + } else { + SampleRemoval::Keep + }, + }); + + remove_entries |= do_remove; + + // Update trainer stats + if is_spam { + spam_count += 1; + } else { + ham_count += 1; + } } - - samples.push(TrainingTask { - id, - sample, - is_spam, - is_replay: false, - remove: do_remove.then_some(until), - }); - - remove_entries |= do_remove; - - // Update trainer stats - if is_spam { - spam_count += 1; - } else { - ham_count += 1; + Entry::Occupied(entry) => { + let remove = if *entry.get() == Some(until) { + SampleRemoval::Item + } else { + SampleRemoval::ItemAndLink { until } + }; + duplicate_samples.push(TrainingTask { + id, + sample, + is_spam, + is_replay: false, + remove, + }); + remove_entries = true; } - } else { - duplicate_samples.push(TrainingTask { - id, - sample, - is_spam, - is_replay: false, - remove: Some(until), - }); - remove_entries = true; } trainer.last_id = trainer.last_id.max(id); @@ -315,7 +336,7 @@ impl SpamClassifier for Server { sample: sample.clone(), is_spam: false, is_replay: true, - remove: None, + remove: SampleRemoval::Keep, }), ); } else if ham_count > spam_count { @@ -329,7 +350,7 @@ impl SpamClassifier for Server { sample: sample.clone(), is_spam: true, is_replay: true, - remove: None, + remove: SampleRemoval::Keep, }), ); } @@ -884,33 +905,38 @@ async fn delete_samples( let object_id = ObjectType::SpamTrainingSample.to_id(); let mut batch = BatchBuilder::new(); for sample in samples.into_iter().chain(duplicate_samples) { - if let Some(until) = sample.remove { - batch - .with_account_id(sample.sample.account_id) - .clear(BlobOp::Link { - hash: sample.sample.hash, - to: BlobLink::Temporary { until }, - }) - .clear(ValueClass::Registry(RegistryClass::Item { - object_id, - item_id: sample.id, - })) - .clear(ValueClass::Registry(RegistryClass::Index { - index_id: Property::AccountId.to_id(), - object_id, - item_id: sample.id, - key: (sample.sample.account_id as u64).serialize(), - })); + let until = match sample.remove { + SampleRemoval::Keep => continue, + SampleRemoval::Item => None, + SampleRemoval::ItemAndLink { until } => Some(until), + }; + batch.with_account_id(sample.sample.account_id); + if let Some(until) = until { + batch.clear(BlobOp::Link { + hash: sample.sample.hash, + to: BlobLink::Temporary { until }, + }); + } + batch + .clear(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: sample.id, + })) + .clear(ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id: sample.id, + key: (sample.sample.account_id as u64).serialize(), + })); - if batch.is_large_batch() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - batch.with_account_id(sample.sample.account_id); - } + if batch.is_large_batch() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + batch.with_account_id(sample.sample.account_id); } } if !batch.is_empty() { diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index e0e4f14..f0b83a7 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -11,7 +11,14 @@ use common::{ config::mailstore::spamfilter::{DnsBlServer, Element, IpResolver, Location}, expr::functions::ResolveVariable, }; -use mail_auth::{Error, common::resolver::ToFqdn}; +use mail_auth::common::resolver::ToFqdn; +#[cfg(not(feature = "test_mode"))] +use mail_auth::hickory_resolver::{ + net::{DnsError, NetError}, + proto::rr::{Name, RData}, +}; +#[cfg(feature = "test_mode")] +use mail_auth::{DnsError, Error}; use std::{ net::Ipv4Addr, sync::Arc, @@ -19,6 +26,18 @@ use std::{ }; use trc::SpamEvent; +const MAX_NEGATIVE_TTL: u32 = 3600; + +enum DnsblAnswer { + Listed { + ips: Vec, + expires: Instant, + }, + NotListed { + expires: Option, + }, +} + pub(crate) async fn check_dnsbl( server: &Server, ctx: &mut SpamFilterContext<'_>, @@ -49,7 +68,7 @@ pub(crate) async fn check_dnsbl( for dnsbl in &server.core.spam.dnsbl.servers { if dnsbl.scope == scope && checks < max_checks - && let Some(tag) = is_dnsbl( + && let Some(codes) = dnsbl_codes( server, dnsbl, SpamFilterResolver::new(ctx, resolver, location), @@ -58,7 +77,19 @@ pub(crate) async fn check_dnsbl( ) .await { - ctx.result.add_tag(tag); + for code in codes.iter() { + let tag = server + .eval_if::( + &dnsbl.tags, + &SpamFilterResolver::new(ctx, code, location), + ctx.input.span_id, + ) + .await; + + if let Some(tag) = tag { + ctx.result.add_tag(tag); + } + } } } @@ -71,13 +102,13 @@ pub(crate) async fn check_dnsbl( } } -async fn is_dnsbl( +async fn dnsbl_codes( server: &Server, config: &DnsBlServer, resolver: SpamFilterResolver<'_, impl ResolveVariable>, element: Element, checks: &mut usize, -) -> Option { +) -> Option> { let time = Instant::now(); let zone = server .eval_if::(&config.zone, &resolver, resolver.ctx.input.span_id) @@ -92,105 +123,122 @@ async fn is_dnsbl( { None } else { - server - .eval_if( - &config.tags, - &SpamFilterResolver::new( - resolver.ctx, - &IpResolver::new( - format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(), - ), - resolver.location, - ), - resolver.ctx.input.span_id, - ) - .await + Some(Arc::from([IpResolver::new( + format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(), + )])) }; } } - let result = match server.inner.cache.dns_rbl.get(zone.as_str()) { - Some(Some(result)) => result, - Some(None) => return None, - None => { - *checks += 1; + if let Some(codes) = server.inner.cache.dns_rbl.get(zone.as_str()) { + return codes; + } - match server - .core - .smtp - .resolvers - .dns - .ipv4_lookup_raw(zone.to_fqdn().as_ref()) - .await - { - Ok(result) => { - trc::event!( - Spam(SpamEvent::Dnsbl), - Hostname = zone.clone(), - Result = result - .entry - .iter() - .map(|ip| trc::Value::from(ip.to_string())) - .collect::>(), - Details = element.as_str(), - Elapsed = time.elapsed() - ); + *checks += 1; - let entry = Arc::new(IpResolver::new( - result - .entry - .iter() - .copied() - .next() - .unwrap_or(Ipv4Addr::BROADCAST) - .into(), - )); + match resolve_zone(server, zone.to_fqdn().as_ref()).await { + Ok(DnsblAnswer::Listed { ips, expires }) => { + trc::event!( + Spam(SpamEvent::Dnsbl), + Hostname = zone.clone(), + Result = ips + .iter() + .map(|ip| trc::Value::from(ip.to_string())) + .collect::>(), + Details = element.as_str(), + Elapsed = time.elapsed() + ); - server.inner.cache.dns_rbl.insert_with_expiry( - zone.into(), - Some(entry.clone()), - result.expires, - ); + let codes: Arc<[IpResolver]> = ips + .into_iter() + .map(|ip| IpResolver::new(ip.into())) + .collect(); - entry - } - Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => { - trc::event!( - Spam(SpamEvent::Dnsbl), - Hostname = zone.clone(), - Result = trc::Value::None, - Details = element.as_str(), - Elapsed = time.elapsed() - ); + server.inner.cache.dns_rbl.insert_with_expiry( + zone.into(), + Some(codes.clone()), + expires, + ); - server.inner.cache.dns_rbl.insert( - zone.into(), - None, - Duration::from_secs(86400), - ); - - return None; - } - Err(err) => { - trc::event!( - Spam(SpamEvent::DnsblError), - Hostname = zone, - Elapsed = time.elapsed(), - Details = element.as_str(), - CausedBy = err.to_string() - ); - - return None; - } - } + Some(codes) } - }; + Ok(DnsblAnswer::NotListed { expires }) => { + trc::event!( + Spam(SpamEvent::Dnsbl), + Hostname = zone.clone(), + Result = trc::Value::None, + Details = element.as_str(), + Elapsed = time.elapsed() + ); - server - .eval_if( - &config.tags, - &SpamFilterResolver::new(resolver.ctx, result.as_ref(), resolver.location), - resolver.ctx.input.span_id, - ) - .await + if let Some(expires) = expires { + server + .inner + .cache + .dns_rbl + .insert_with_expiry(zone.into(), None, expires); + } + + None + } + Err(err) => { + trc::event!( + Spam(SpamEvent::DnsblError), + Hostname = zone, + Elapsed = time.elapsed(), + Details = element.as_str(), + CausedBy = err + ); + + None + } + } +} + +#[cfg(not(feature = "test_mode"))] +async fn resolve_zone(server: &Server, zone: &str) -> Result { + let name = Name::from_str_relaxed(zone).map_err(|err| err.to_string())?; + + match server.core.smtp.resolvers.dns.0.ipv4_lookup(name).await { + Ok(lookup) => { + let expires = lookup.valid_until(); + let ips = lookup + .answers() + .iter() + .filter_map(|record| match &record.data { + RData::A(a) => Some(a.0), + _ => None, + }) + .collect::>(); + + Ok(if !ips.is_empty() { + DnsblAnswer::Listed { ips, expires } + } else { + DnsblAnswer::NotListed { + expires: Some(expires), + } + }) + } + Err(NetError::Dns(DnsError::NoRecordsFound(no_records))) => Ok(DnsblAnswer::NotListed { + expires: no_records + .negative_ttl + .filter(|ttl| *ttl > 0) + .map(|ttl| Instant::now() + Duration::from_secs(ttl.min(MAX_NEGATIVE_TTL).into())), + }), + Err(err) => Err(err.to_string()), + } +} + +#[cfg(feature = "test_mode")] +async fn resolve_zone(server: &Server, zone: &str) -> Result { + match server.core.smtp.resolvers.dns.ipv4_lookup_raw(zone).await { + Ok(result) => Ok(DnsblAnswer::Listed { + ips: result.entry.to_vec(), + expires: result.expires, + }), + Err(Error::Dns(DnsError::RecordNotFound(_))) => Ok(DnsblAnswer::NotListed { + expires: Some(Instant::now() + Duration::from_secs(MAX_NEGATIVE_TTL.into())), + }), + Err(err) => Err(err.to_string()), + } } diff --git a/crates/spam-filter/src/modules/pyzor.rs b/crates/spam-filter/src/modules/pyzor.rs index 6ee2056..8f3834a 100644 --- a/crates/spam-filter/src/modules/pyzor.rs +++ b/crates/spam-filter/src/modules/pyzor.rs @@ -33,17 +33,9 @@ pub(crate) async fn pyzor_check( message: &Message<'_>, config: &PyzorConfig, ) -> trc::Result> { - // Make sure there is at least one text part - if !message - .parts - .iter() - .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_))) - { + let Some(request) = message.pyzor_check_message() else { return Ok(None); - } - - // Hash message - let request = message.pyzor_check_message(); + }; // Send message to address. inbuxa: in tests, a fixed table answers // instead of a public server (test_response). @@ -167,15 +159,15 @@ impl PyzorWrite for Sha1 { } trait PyzorDigest { - fn pyzor_digest(&self, writer: W) -> W; + fn pyzor_digest(&self, writer: W) -> Option; } pub trait PyzorCheck { - fn pyzor_check_message(&self) -> String; + fn pyzor_check_message(&self) -> Option; } impl PyzorDigest for Message<'_> { - fn pyzor_digest(&self, writer: W) -> W { + fn pyzor_digest(&self, writer: W) -> Option { let parts = self .parts .iter() @@ -191,7 +183,7 @@ impl PyzorDigest for Message<'_> { } impl PyzorCheck for Message<'_> { - fn pyzor_check_message(&self) -> String { + fn pyzor_check_message(&self) -> Option { let time = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); @@ -204,9 +196,9 @@ impl PyzorCheck for Message<'_> { } } -fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String { +fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> Option { // Hash message - let hash = message.pyzor_digest(Sha1::new()).finalize().hex_encode(); + let hash = message.pyzor_digest(Sha1::new())?.finalize().hex_encode(); // Hash key let mut hash_key = Sha1::new(); hash_key.update("anonymous:".as_bytes()); @@ -226,10 +218,10 @@ fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String sig.update(format!(":{time}:{hash_key}")); let sig = sig.finalize().hex_encode(); - format!("{message}\nSig: {sig}\n") + Some(format!("{message}\nSig: {sig}\n")) } -fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> W +fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> Option where I: Iterator, W: PyzorWrite, @@ -291,6 +283,10 @@ where } } + if result.is_empty() { + return None; + } + if result.len() > ATOMIC_NUM_LINES { for (offset, length) in DIGEST_SPEC { for i in 0..*length { @@ -305,7 +301,7 @@ where } } - writer + Some(writer) } fn html_to_text(input: &str) -> String { @@ -489,7 +485,8 @@ mod test { &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(), 1697468672, 49005, - ); + ) + .unwrap(); assert_eq!( message, @@ -522,10 +519,9 @@ mod test { "http://spammer.com/special-offers?buy=now", ] { assert_eq!( - String::from_utf8(pyzor_digest( - Vec::new(), - format!("Test {strip_me} Test2").lines(), - )) + String::from_utf8( + pyzor_digest(Vec::new(), format!("Test {strip_me} Test2").lines()).unwrap() + ) .unwrap(), "TestTest2" ); @@ -533,20 +529,26 @@ mod test { // Test short lines assert_eq!( - String::from_utf8(pyzor_digest( - Vec::new(), - concat!("This line is included\n", "not this\n", "This also").lines(), - )) + String::from_utf8( + pyzor_digest( + Vec::new(), + concat!("This line is included\n", "not this\n", "This also").lines(), + ) + .unwrap() + ) .unwrap(), "ThislineisincludedThisalso" ); // Test atomic assert_eq!( - String::from_utf8(pyzor_digest( - Vec::new(), - "All this message\nShould be included\nIn the digest".lines(), - )) + String::from_utf8( + pyzor_digest( + Vec::new(), + "All this message\nShould be included\nIn the digest".lines(), + ) + .unwrap() + ) .unwrap(), "AllthismessageShouldbeincludedInthedigest" ); @@ -561,7 +563,7 @@ mod test { expected += format!("Line{i}testtesttest").as_str(); } assert_eq!( - String::from_utf8(pyzor_digest(Vec::new(), text.lines(),)).unwrap(), + String::from_utf8(pyzor_digest(Vec::new(), text.lines()).unwrap()).unwrap(), expected ); @@ -593,7 +595,8 @@ mod test { MessageParser::new() .parse(input) .unwrap() - .pyzor_digest(Vec::new(),) + .pyzor_digest(Vec::new()) + .unwrap() ) .unwrap(), expected, @@ -606,7 +609,8 @@ mod test { MessageParser::new() .parse(HTML_TEXT_STYLE_SCRIPT) .unwrap() - .pyzor_digest(Sha1::new(),) + .pyzor_digest(Sha1::new()) + .unwrap() .finalize() .hex_encode(), "b2c27325a034c581df0c9ef37e4a0d63208a3e7e", diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index ba1c06d..0a421fc 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "store" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 7c8fa1c..798edf5 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -26,6 +26,8 @@ const CHURN_DELETION_WINDOW: usize = 4096; const CHURN_DELETION_TRIGGER: usize = 1024; const CHURN_DELETION_RATIO: f64 = 0.5; const BYTES_PER_SYNC: u64 = 1024 * 1024; +const MAX_LOG_FILE_SIZE: usize = 10 * 1024 * 1024; +const KEEP_LOG_FILE_NUM: usize = 5; #[derive(Clone, Copy)] enum CfProfile { @@ -118,6 +120,8 @@ impl RocksDbStore { .set_db_write_buffer_size((config.buffer_size as usize).max(MIN_DB_WRITE_BUFFER_SIZE)); db_opts.set_bytes_per_sync(BYTES_PER_SYNC); db_opts.set_wal_bytes_per_sync(BYTES_PER_SYNC); + db_opts.set_max_log_file_size(MAX_LOG_FILE_SIZE); + db_opts.set_keep_log_file_num(KEEP_LOG_FILE_NUM); Ok(Store::RocksDb(Arc::new(RocksDbStore { db: OptimisticTransactionDB::open_cf_descriptors(&db_opts, idx_path, cfs) diff --git a/crates/trc/Cargo.toml b/crates/trc/Cargo.toml index 6ea1a51..68dfd0a 100644 --- a/crates/trc/Cargo.toml +++ b/crates/trc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "trc" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/trc/event-macro/Cargo.toml b/crates/trc/event-macro/Cargo.toml index f492927..094ad47 100644 --- a/crates/trc/event-macro/Cargo.toml +++ b/crates/trc/event-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "event_macro" -version = "0.16.24" +version = "0.16.25" edition = "2024" [lib] diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 56184db..70ce449 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "types" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 7433b48..b9fd933 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "utils" -version = "0.16.24" +version = "0.16.25" edition = "2024" [dependencies] diff --git a/crates/utils/proc-macros/Cargo.toml b/crates/utils/proc-macros/Cargo.toml index 93499de..85ae8e4 100644 --- a/crates/utils/proc-macros/Cargo.toml +++ b/crates/utils/proc-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "proc_macros" -version = "0.16.24" +version = "0.16.25" edition = "2024" [lib] diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 5a15c5e..04c18f0 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tests" -version = "0.16.24" +version = "0.16.25" edition = "2024" [features] diff --git a/tests/resources/smtp/antispam/combined.test b/tests/resources/smtp/antispam/combined.test index ee411e8..ddbc3d4 100644 --- a/tests/resources/smtp/antispam/combined.test +++ b/tests/resources/smtp/antispam/combined.test @@ -6,8 +6,8 @@ spf.result none spf_ehlo.result none dmarc.result none remote_ip 195.210.29.48 -expect_header X-Spam-Result: ARC_NA (0.00), DKIM2_NA (0.00), DKIM_NA (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_DATA_URI (0.00), HAS_LINK_TO_LARGE_IMG (0.00), HTML_SHORT_1 (0.00), MID_RHS_MATCH_ENV_FROM (0.00), RCPT_COUNT_ONE (0.00), SPF_NA (0.00), SUBJECT_ENDS_EXCLAIM (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_COUNT_ZERO (0.10), RCVD_NO_TLS_LAST (0.10), MIME_HTML_ONLY (0.20), HELO_NORES_A_OR_MX (0.30), AUTH_NA (1.00), DATE_IN_PAST (1.00), DMARC_NA (1.00), MID_RHS_MATCH_FROM (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), RDNS_NONE (2.00), PYZOR (3.50) -expect_header X-Spam-Score: spam, score=13.70 +expect_header X-Spam-Result: ARC_NA (0.00), DKIM2_NA (0.00), DKIM_NA (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_DATA_URI (0.00), HAS_LINK_TO_LARGE_IMG (0.00), HTML_SHORT_1 (0.00), MID_RHS_MATCH_ENV_FROM (0.00), RCPT_COUNT_ONE (0.00), SPF_NA (0.00), SUBJECT_ENDS_EXCLAIM (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_COUNT_ZERO (0.10), RCVD_NO_TLS_LAST (0.10), MIME_HTML_ONLY (0.20), HELO_NORES_A_OR_MX (0.30), AUTH_NA (1.00), DATE_IN_PAST (1.00), DMARC_NA (1.00), MID_RHS_MATCH_FROM (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), RDNS_NONE (2.00) +expect_header X-Spam-Score: spam, score=10.20 From: Client Services To: licensing@stalw.art diff --git a/tests/resources/smtp/antispam/rbl.test b/tests/resources/smtp/antispam/rbl.test index 4999a0c..b90a0f1 100644 --- a/tests/resources/smtp/antispam/rbl.test +++ b/tests/resources/smtp/antispam/rbl.test @@ -38,6 +38,14 @@ My e-mail is spammer@sem-uribl.com And my website is https://sem-fresh15.com/offers.html Try cheating with a trusted domain user@dkimtrusted.org + +expect DBL_SPAM DBL_PHISH + +From: user@example.net +Subject: test + +Our website is https://dbl-multi.com/offers.html + expect DBL_MALWARE diff --git a/tests/src/automation/dkim.rs b/tests/src/automation/dkim.rs index 4963a06..e645382 100644 --- a/tests/src/automation/dkim.rs +++ b/tests/src/automation/dkim.rs @@ -10,20 +10,37 @@ use common::{config::smtp::auth::Dkim1Signer, network::dns::update::DNS_RECORDS} use dns_update::{DnsRecord, NamedDnsRecord}; use registry::{ schema::{ - enums::{DkimRotationStage, DnsRecordType}, - prelude::ObjectType, + enums::{DkimRotationStage, DnsRecordType, IpProtocol, TsigAlgorithm}, + prelude::{ObjectType, Property}, structs::{ CertificateManagement, Dkim1Signature, DkimManagement, DkimManagementProperties, DkimSignature, DnsManagement, DnsManagementProperties, DnsServer, DnsServerCloudflare, - Domain, SecretKey, SecretKeyValue, + DnsServerTsig, Domain, SecretKey, SecretKeyValue, Task, TaskDomainManagement, + TaskManager, TaskRetryStrategy, TaskRetryStrategyFixed, TaskStatus, }, }, - types::duration::Duration, + types::{duration::Duration, map::Map}, }; +use serde_json::json; use store::write::now; use types::id::Id; +const SHORT_ROTATION_MS: u64 = 6_000; +const LONG_ROTATION_MS: u64 = 3_600_000; + pub async fn test(test: &TestServer) { + fast_retry_tests(test).await; + unscheduled_keys_test(test, "dkim-manual.org", |_| DnsManagement::Manual).await; + unscheduled_keys_test(test, "dkim-unpublished.org", |dns_server_id| { + DnsManagement::Automatic(DnsManagementProperties { + dns_server_id, + publish_records: Map::new(vec![DnsRecordType::Spf]), + ..Default::default() + }) + }) + .await; + automatic_to_manual_dns_test(test).await; + println!("Running DKIM Management tests..."); let account = test.account("admin@example.org"); DNS_RECORDS.lock().unwrap().clear(); @@ -98,8 +115,8 @@ pub async fn test(test: &TestServer) { // Make sure the DNS records were created let records = DNS_RECORDS.lock().unwrap().clone(); - assert_key_has_dns_record(&records, &rot1_signatures.v1_rsa[0]); - assert_key_has_dns_record(&records, &rot1_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_ed25519[0]); // Expect a rotation to happen and new keys to be created let rot2_signatures = account @@ -111,10 +128,10 @@ pub async fn test(test: &TestServer) { // Make sure both old and new keys have DNS records let records = DNS_RECORDS.lock().unwrap().clone(); - assert_key_has_dns_record(&records, &rot1_signatures.v1_rsa[0]); - assert_key_has_dns_record(&records, &rot1_signatures.v1_ed25519[0]); - assert_key_has_dns_record(&records, &rot2_signatures.v1_rsa[0]); - assert_key_has_dns_record(&records, &rot2_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot1_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_ed25519[0]); // Make sure only the new keys are being used for signing assert_ne!( @@ -145,19 +162,19 @@ pub async fn test(test: &TestServer) { // Make sure the old records were deleted let records = DNS_RECORDS.lock().unwrap().clone(); - assert_key_has_no_dns_record(&records, &rot1_signatures.v1_rsa[0]); - assert_key_has_no_dns_record(&records, &rot1_signatures.v1_ed25519[0]); - assert_key_has_dns_record(&records, &rot2_signatures.v1_rsa[0]); - assert_key_has_dns_record(&records, &rot2_signatures.v1_ed25519[0]); - assert_key_has_dns_record(&records, &rot3_signatures.v1_rsa[0]); - assert_key_has_dns_record(&records, &rot3_signatures.v1_ed25519[0]); + assert_key_has_no_dns_record(&records, "dkim.org", &rot1_signatures.v1_rsa[0]); + assert_key_has_no_dns_record(&records, "dkim.org", &rot1_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot2_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot3_signatures.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot3_signatures.v1_ed25519[0]); // Make sure the DNS management task does not republish the retired keys let (published, zone_file) = test.published_dkim_records(domain_id).await; - assert_key_has_no_dns_record(&published, &rot1_signatures.v1_rsa[0]); - assert_key_has_no_dns_record(&published, &rot1_signatures.v1_ed25519[0]); - assert_key_has_dns_record(&published, &rot3_signatures.v1_rsa[0]); - assert_key_has_dns_record(&published, &rot3_signatures.v1_ed25519[0]); + assert_key_has_no_dns_record(&published, "dkim.org", &rot1_signatures.v1_rsa[0]); + assert_key_has_no_dns_record(&published, "dkim.org", &rot1_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&published, "dkim.org", &rot3_signatures.v1_rsa[0]); + assert_key_has_dns_record(&published, "dkim.org", &rot3_signatures.v1_ed25519[0]); assert_zone_file_omits_key(&zone_file, &rot1_signatures.v1_rsa[0]); assert_zone_file_omits_key(&zone_file, &rot1_signatures.v1_ed25519[0]); @@ -192,17 +209,17 @@ pub async fn test(test: &TestServer) { // Make sure the old records were updated let records = DNS_RECORDS.lock().unwrap().clone(); - assert_key_has_dns_record(&records, &rot4_signatures.v1_rsa[0]); - assert_key_has_dns_record(&records, &rot4_signatures.v1_ed25519[0]); - assert_key_has_no_dns_record(&records, &rot2_signatures.v1_rsa[0]); - assert_key_has_no_dns_record(&records, &rot2_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot4_signatures.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim.org", &rot4_signatures.v1_ed25519[0]); + assert_key_has_no_dns_record(&records, "dkim.org", &rot2_signatures.v1_rsa[0]); + assert_key_has_no_dns_record(&records, "dkim.org", &rot2_signatures.v1_ed25519[0]); // Make sure the DNS management task does not republish the retired keys let (published, zone_file) = test.published_dkim_records(domain_id).await; - assert_key_has_no_dns_record(&published, &rot2_signatures.v1_rsa[0]); - assert_key_has_no_dns_record(&published, &rot2_signatures.v1_ed25519[0]); - assert_key_has_dns_record(&published, &rot4_signatures.v1_rsa[0]); - assert_key_has_dns_record(&published, &rot4_signatures.v1_ed25519[0]); + assert_key_has_no_dns_record(&published, "dkim.org", &rot2_signatures.v1_rsa[0]); + assert_key_has_no_dns_record(&published, "dkim.org", &rot2_signatures.v1_ed25519[0]); + assert_key_has_dns_record(&published, "dkim.org", &rot4_signatures.v1_rsa[0]); + assert_key_has_dns_record(&published, "dkim.org", &rot4_signatures.v1_ed25519[0]); assert_zone_file_omits_key(&zone_file, &rot2_signatures.v1_rsa[0]); assert_zone_file_omits_key(&zone_file, &rot2_signatures.v1_ed25519[0]); @@ -235,6 +252,470 @@ pub async fn test(test: &TestServer) { account.registry_destroy_all(ObjectType::DnsServer).await; } +async fn fast_retry_tests(test: &TestServer) { + let account = test.account("admin@example.org"); + + // Retry failed tasks every second + account + .registry_update_setting( + TaskManager { + max_attempts: 100, + strategy: TaskRetryStrategy::FixedDelay(TaskRetryStrategyFixed { + delay: 1_000u64.into(), + }), + total_deadline: 86_400_000u64.into(), + }, + &[], + ) + .await; + account.reload_settings().await; + + failed_publish_test(test).await; + manual_dns_pending_test(test).await; + + account + .registry_update_setting(TaskManager::default(), &[]) + .await; + account.reload_settings().await; +} + +async fn failed_publish_test(test: &TestServer) { + println!("Running DKIM failed publish tests..."); + let account = test.account("admin@example.org"); + DNS_RECORDS.lock().unwrap().clear(); + account.dkim_signatures().await.assert_total(0, 0); + + // Create an in-memory DNS server and a DNS server that refuses connections + let dns_server_id = account.create_memory_dns_server().await; + let failing_dns_server_id = account.create_failing_dns_server().await; + + // Create a domain whose initial keys rotate shortly + let domain_id = account + .registry_create_object(Domain { + name: "dkim-retry.org".to_string(), + certificate_management: CertificateManagement::Manual, + dkim_management: dkim_management(SHORT_ROTATION_MS), + dns_management: dns_management(dns_server_id), + ..Default::default() + }) + .await; + let initial = account + .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)]) + .await + .assert_total(1, 1); + let old_rsa = initial.v1_rsa[0].selector.clone(); + let old_ed = initial.v1_ed25519[0].selector.clone(); + test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed]) + .await; + + // Point the domain at the failing DNS server before the rotation is due, and + // make the keys created from now on long-lived + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: dns_management(failing_dns_server_id), + Property::DkimManagement: dkim_management(LONG_ROTATION_MS), + }), + ) + .await; + assert!( + initial.v1_rsa[0].next_transition_at.unwrap().timestamp() > now() as i64, + "Rotation was due before the DNS server could be replaced: {:#?}", + initial + ); + + // The new keys cannot be published, so they must stay pending while the + // old keys remain active and keep signing + let failed = account + .wait_for_dkim_stages(&[ + (DkimRotationStage::Pending, 2), + (DkimRotationStage::Active, 2), + ]) + .await + .assert_total(2, 2) + .assert_selector_stage(&old_rsa, DkimRotationStage::Active) + .assert_selector_stage(&old_ed, DkimRotationStage::Active); + let new_rsa = failed.v1_rsa[0].selector.clone(); + let new_ed = failed.v1_ed25519[0].selector.clone(); + let failed = failed + .assert_selector_stage(&new_rsa, DkimRotationStage::Pending) + .assert_selector_stage(&new_ed, DkimRotationStage::Pending); + test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed]) + .await; + + // Several retries must neither create duplicate keys nor retire the old ones + let failure_reason = account.wait_for_dkim_task_attempts(domain_id, 4).await; + assert!( + failure_reason.contains("Failed to publish DKIM record"), + "Unexpected failure reason: {failure_reason}" + ); + let retried = account.dkim_signatures().await; + assert_eq!( + retried, failed, + "DKIM signatures changed while the DNS server was failing" + ); + test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed]) + .await; + + // Under manual DNS management the pending keys stay pending next to the + // active keys, and the task stops retrying + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: DnsManagement::Manual, + }), + ) + .await; + account.wait_for_no_dkim_tasks(domain_id).await; + assert_eq!( + account.dkim_signatures().await, + failed, + "DKIM signatures changed under manual DNS management" + ); + test.assert_has_signers("dkim-retry.org", &[&old_rsa, &old_ed]) + .await; + + // Once automatic DNS management uses a working server again, the pending + // keys are activated and the old keys start retiring + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: dns_management(dns_server_id), + }), + ) + .await; + let recovered = account + .wait_for_dkim_stages(&[ + (DkimRotationStage::Active, 2), + (DkimRotationStage::Retiring, 2), + ]) + .await + .assert_total(2, 2) + .assert_selector_stage(&new_rsa, DkimRotationStage::Active) + .assert_selector_stage(&new_ed, DkimRotationStage::Active) + .assert_selector_stage(&old_rsa, DkimRotationStage::Retiring) + .assert_selector_stage(&old_ed, DkimRotationStage::Retiring); + test.assert_has_signers("dkim-retry.org", &[&new_rsa, &new_ed]) + .await; + let records = DNS_RECORDS.lock().unwrap().clone(); + assert_key_has_dns_record(&records, "dkim-retry.org", &recovered.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim-retry.org", &recovered.v1_ed25519[0]); + + // Cleanup + account.registry_destroy_all(ObjectType::Task).await; + account + .registry_destroy_all(ObjectType::DkimSignature) + .await; + account + .registry_destroy(ObjectType::Domain, [domain_id]) + .await + .assert_destroyed(&[domain_id]); + account.registry_destroy_all(ObjectType::DnsServer).await; +} + +async fn unscheduled_keys_test( + test: &TestServer, + domain: &str, + initial_dns_management: fn(Id) -> DnsManagement, +) { + println!("Running DKIM unscheduled key tests for {domain}..."); + let account = test.account("admin@example.org"); + DNS_RECORDS.lock().unwrap().clear(); + account.dkim_signatures().await.assert_total(0, 0); + let dns_server_id = account.create_memory_dns_server().await; + + // Keys created while DKIM records are not published automatically are + // active, unpublished and have no rotation schedule + let domain_id = account + .registry_create_object(Domain { + name: domain.to_string(), + certificate_management: CertificateManagement::Manual, + dkim_management: dkim_management(SHORT_ROTATION_MS), + dns_management: initial_dns_management(dns_server_id), + ..Default::default() + }) + .await; + let initial = account + .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)]) + .await + .assert_total(1, 1); + assert!( + initial.keys().all(|key| key.next_transition_at.is_none()), + "Unexpected rotation schedule for unpublished keys: {initial:#?}" + ); + let records = DNS_RECORDS.lock().unwrap().clone(); + assert_key_has_no_dns_record(&records, domain, &initial.v1_rsa[0]); + assert_key_has_no_dns_record(&records, domain, &initial.v1_ed25519[0]); + let old_rsa = initial.v1_rsa[0].selector.clone(); + let old_ed = initial.v1_ed25519[0].selector.clone(); + + // Publishing DKIM records automatically publishes the keys and schedules + // their rotation + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: dns_management(dns_server_id), + }), + ) + .await; + let scheduled = account + .wait_for_dkim("active keys with a rotation schedule", |signatures| { + signatures.total() == 2 + && signatures.stage_count(DkimRotationStage::Active) == 2 + && signatures + .keys() + .all(|key| key.next_transition_at.is_some()) + }) + .await; + + // Make the keys created by the rotation long-lived + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DkimManagement: dkim_management(LONG_ROTATION_MS), + }), + ) + .await; + assert!( + scheduled + .keys() + .all(|key| key.next_transition_at.unwrap().timestamp() > now() as i64), + "Rotation was due before the rotation period could be extended: {scheduled:#?}" + ); + wait_for_dns_records(domain, &[&old_rsa, &old_ed]).await; + + // The keys rotate once the schedule elapses + let rotated = account + .wait_for_dkim_stages(&[ + (DkimRotationStage::Active, 2), + (DkimRotationStage::Retiring, 2), + ]) + .await + .assert_total(2, 2) + .assert_selector_stage(&old_rsa, DkimRotationStage::Retiring) + .assert_selector_stage(&old_ed, DkimRotationStage::Retiring); + test.assert_has_signers( + domain, + &[&rotated.v1_rsa[0].selector, &rotated.v1_ed25519[0].selector], + ) + .await; + let records = DNS_RECORDS.lock().unwrap().clone(); + assert_key_has_dns_record(&records, domain, &rotated.v1_rsa[0]); + assert_key_has_dns_record(&records, domain, &rotated.v1_ed25519[0]); + + // Cleanup + account.registry_destroy_all(ObjectType::Task).await; + account + .registry_destroy_all(ObjectType::DkimSignature) + .await; + account + .registry_destroy(ObjectType::Domain, [domain_id]) + .await + .assert_destroyed(&[domain_id]); + account.registry_destroy_all(ObjectType::DnsServer).await; +} + +async fn automatic_to_manual_dns_test(test: &TestServer) { + println!("Running DKIM automatic to manual DNS tests..."); + let account = test.account("admin@example.org"); + DNS_RECORDS.lock().unwrap().clear(); + account.dkim_signatures().await.assert_total(0, 0); + let dns_server_id = account.create_memory_dns_server().await; + + // Create a domain whose initial keys rotate shortly + let domain_id = account + .registry_create_object(Domain { + name: "dkim-mirror.org".to_string(), + certificate_management: CertificateManagement::Manual, + dkim_management: dkim_management(SHORT_ROTATION_MS), + dns_management: dns_management(dns_server_id), + ..Default::default() + }) + .await; + let initial = account + .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)]) + .await + .assert_total(1, 1); + let old_rsa = initial.v1_rsa[0].selector.clone(); + let old_ed = initial.v1_ed25519[0].selector.clone(); + + // Switch to manual DNS management before the rotation is due, and make the + // keys created from now on long-lived + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: DnsManagement::Manual, + Property::DkimManagement: dkim_management(LONG_ROTATION_MS), + }), + ) + .await; + let due = initial.v1_rsa[0].next_transition_at.unwrap().timestamp(); + let wait_secs = due - now() as i64; + assert!( + wait_secs > 0, + "Rotation was due before DNS management was switched: {initial:#?}" + ); + + // Once the rotation is due, the task must neither rotate the keys nor keep + // retrying + tokio::time::sleep(std::time::Duration::from_secs(wait_secs as u64 + 1)).await; + account.wait_for_no_dkim_tasks(domain_id).await; + assert_eq!( + account.dkim_signatures().await, + initial, + "DKIM signatures changed under manual DNS management" + ); + test.assert_has_signers("dkim-mirror.org", &[&old_rsa, &old_ed]) + .await; + + // Switching back to automatic DNS management completes the overdue rotation + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: dns_management(dns_server_id), + }), + ) + .await; + let rotated = account + .wait_for_dkim_stages(&[ + (DkimRotationStage::Active, 2), + (DkimRotationStage::Retiring, 2), + ]) + .await + .assert_total(2, 2) + .assert_selector_stage(&old_rsa, DkimRotationStage::Retiring) + .assert_selector_stage(&old_ed, DkimRotationStage::Retiring); + test.assert_has_signers( + "dkim-mirror.org", + &[&rotated.v1_rsa[0].selector, &rotated.v1_ed25519[0].selector], + ) + .await; + let records = DNS_RECORDS.lock().unwrap().clone(); + assert_key_has_dns_record(&records, "dkim-mirror.org", &rotated.v1_rsa[0]); + assert_key_has_dns_record(&records, "dkim-mirror.org", &rotated.v1_ed25519[0]); + + // Cleanup + account.registry_destroy_all(ObjectType::Task).await; + account + .registry_destroy_all(ObjectType::DkimSignature) + .await; + account + .registry_destroy(ObjectType::Domain, [domain_id]) + .await + .assert_destroyed(&[domain_id]); + account.registry_destroy_all(ObjectType::DnsServer).await; +} + +async fn manual_dns_pending_test(test: &TestServer) { + println!("Running DKIM pending key under manual DNS tests..."); + let account = test.account("admin@example.org"); + DNS_RECORDS.lock().unwrap().clear(); + account.dkim_signatures().await.assert_total(0, 0); + let failing_dns_server_id = account.create_failing_dns_server().await; + + // Keys created while the DNS server is failing stay pending + let domain_id = account + .registry_create_object(Domain { + name: "dkim-pending.org".to_string(), + certificate_management: CertificateManagement::Manual, + dkim_management: dkim_management(LONG_ROTATION_MS), + dns_management: dns_management(failing_dns_server_id), + ..Default::default() + }) + .await; + let pending = account + .wait_for_dkim_stages(&[(DkimRotationStage::Pending, 2)]) + .await + .assert_total(1, 1); + let rsa = pending.v1_rsa[0].selector.clone(); + let ed = pending.v1_ed25519[0].selector.clone(); + + // Switching to manual DNS management activates the pending keys without a + // rotation schedule, and the task stops retrying + account + .registry_update_object( + ObjectType::Domain, + domain_id, + json!({ + Property::DnsManagement: DnsManagement::Manual, + }), + ) + .await; + let active = account + .wait_for_dkim_stages(&[(DkimRotationStage::Active, 2)]) + .await + .assert_total(1, 1) + .assert_selector_stage(&rsa, DkimRotationStage::Active) + .assert_selector_stage(&ed, DkimRotationStage::Active); + assert!( + active.keys().all(|key| key.next_transition_at.is_none()), + "Unexpected rotation schedule under manual DNS management: {active:#?}" + ); + test.assert_has_signers("dkim-pending.org", &[&rsa, &ed]) + .await; + account.wait_for_no_dkim_tasks(domain_id).await; + + // Cleanup + account.registry_destroy_all(ObjectType::Task).await; + account + .registry_destroy_all(ObjectType::DkimSignature) + .await; + account + .registry_destroy(ObjectType::Domain, [domain_id]) + .await + .assert_destroyed(&[domain_id]); + account.registry_destroy_all(ObjectType::DnsServer).await; +} + +fn dns_management(dns_server_id: Id) -> DnsManagement { + DnsManagement::Automatic(DnsManagementProperties { + dns_server_id, + publish_records: Map::new(vec![DnsRecordType::Dkim]), + ..Default::default() + }) +} + +fn dkim_management(rotate_after: u64) -> DkimManagement { + DkimManagement::Automatic(DkimManagementProperties { + delete_after: Duration::from_millis(LONG_ROTATION_MS), + retire_after: Duration::from_millis(LONG_ROTATION_MS), + rotate_after: Duration::from_millis(rotate_after), + selector_template: "dummy-v{version}-{algorithm}-{epoch}".to_string(), + ..Default::default() + }) +} + +async fn wait_for_dns_records(domain: &str, selectors: &[&str]) { + for _ in 0..50 { + let records = DNS_RECORDS.lock().unwrap().clone(); + if selectors + .iter() + .all(|selector| has_dns_record(&records, domain, selector)) + { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + panic!( + "DNS records for selectors {selectors:?} were not published: {:#?}", + DNS_RECORDS.lock().unwrap() + ); +} + #[derive(Debug, PartialEq, Eq, Default)] struct DkimSignatures { v1_rsa: Vec, @@ -265,6 +746,108 @@ impl Account { ); } + async fn wait_for_dkim_stages( + &self, + expected: &[(DkimRotationStage, usize)], + ) -> DkimSignatures { + let expected_total = expected.iter().map(|(_, count)| count).sum::(); + self.wait_for_dkim(&format!("stages {expected:?}"), |signatures| { + signatures.total() == expected_total + && expected + .iter() + .all(|(stage, count)| signatures.stage_count(*stage) == *count) + }) + .await + } + + async fn wait_for_dkim( + &self, + expected: &str, + is_expected: impl Fn(&DkimSignatures) -> bool, + ) -> DkimSignatures { + let mut signatures = self.dkim_signatures().await; + for _ in 0..50 { + if is_expected(&signatures) { + return signatures; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + signatures = self.dkim_signatures().await; + } + panic!("DKIM signatures did not reach {expected}: {signatures:#?}"); + } + + async fn wait_for_no_dkim_tasks(&self, domain_id: Id) { + for _ in 0..60 { + if self.tasks().await.is_some_and(|tasks| { + !tasks.iter().any(|task| { + matches!(&task.task, Task::DkimManagement(task) if task.domain_id == domain_id) + }) + }) { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + panic!( + "DKIM management task did not complete: {:#?}", + self.tasks() + .await + .map(|tasks| tasks.into_iter().map(|task| task.task).collect::>()) + ); + } + + async fn create_failing_dns_server(&self) -> Id { + self.registry_create_object(DnsServer::Tsig(DnsServerTsig { + host: "127.0.0.1".parse().unwrap(), + port: 1, + key_name: "stalwart-update-key".to_string(), + key: SecretKey::Value(SecretKeyValue { + secret: "c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldA==".into(), + }), + protocol: IpProtocol::Tcp, + tsig_algorithm: TsigAlgorithm::HmacSha256, + description: "Unreachable DNS server".to_string(), + timeout: Duration::from_millis(1_000), + ..Default::default() + })) + .await + } + + async fn create_memory_dns_server(&self) -> Id { + self.registry_create_object(DnsServer::Cloudflare(DnsServerCloudflare { + secret: SecretKey::Value(SecretKeyValue { + secret: "test@memory.org".into(), + }), + description: "In-memory DNS server".to_string(), + ..Default::default() + })) + .await + } + + async fn wait_for_dkim_task_attempts(&self, domain_id: Id, attempts: u64) -> String { + for _ in 0..60 { + if let Some(tasks) = self.tasks().await + && let Some(failure_reason) = tasks.into_iter().find_map(|task| match task.task { + Task::DkimManagement(TaskDomainManagement { + domain_id: task_domain_id, + status: TaskStatus::Retry(retry), + }) if task_domain_id == domain_id && retry.attempt_number >= attempts => { + Some(retry.failure_reason) + } + _ => None, + }) + { + return failure_reason; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + panic!( + "DKIM management task did not reach {attempts} attempts: {:#?}", + self.tasks() + .await + .map(|tasks| tasks.into_iter().map(|task| task.task).collect::>()) + ); + } + async fn dkim_signatures(&self) -> DkimSignatures { let signatures = self.registry_get_all::().await; let mut v1_rsa = Vec::new(); @@ -286,9 +869,35 @@ impl Account { } impl DkimSignatures { + fn keys(&self) -> impl Iterator { + self.v1_rsa.iter().chain(&self.v1_ed25519) + } + + fn total(&self) -> usize { + self.v1_rsa.len() + self.v1_ed25519.len() + } + + fn stage_count(&self, stage: DkimRotationStage) -> usize { + self.v1_rsa.iter().filter(|s| s.stage == stage).count() + + self.v1_ed25519.iter().filter(|s| s.stage == stage).count() + } + + fn assert_selector_stage(self, selector: &str, stage: DkimRotationStage) -> Self { + assert!( + self.v1_rsa + .iter() + .chain(self.v1_ed25519.iter()) + .any(|s| s.selector == selector && s.stage == stage), + "Expected selector {} in stage {:?}: {:#?}", + selector, + stage, + self + ); + self + } + fn assert_stage_count(self, stage: DkimRotationStage, count: usize) -> Self { - let actual_count = self.v1_rsa.iter().filter(|s| s.stage == stage).count() - + self.v1_ed25519.iter().filter(|s| s.stage == stage).count(); + let actual_count = self.stage_count(stage); assert_eq!( actual_count, count, "Expected {} signatures in stage {:?}, found {}: {:#?}", @@ -369,21 +978,23 @@ impl TestServer { } } -fn assert_key_has_dns_record(records: &[NamedDnsRecord], key: &Dkim1Signature) { - let expected = format!("{}._domainkey.dkim.org.", key.selector); - for record in records { - if record.name == expected - && let DnsRecord::TXT(txt) = &record.record - && ((key.selector.contains("rsa") && txt.starts_with("v=DKIM1; k=rsa; h=sha256; p=")) - || (key.selector.contains("ed25519") - && txt.starts_with("v=DKIM1; k=ed25519; h=sha256; p="))) - { - return; - } - } - panic!( +fn has_dns_record(records: &[NamedDnsRecord], domain: &str, selector: &str) -> bool { + let expected = format!("{selector}._domainkey.{domain}."); + records.iter().any(|record| { + record.name == expected + && matches!(&record.record, DnsRecord::TXT(txt) + if (selector.contains("rsa") && txt.starts_with("v=DKIM1; k=rsa; h=sha256; p=")) + || (selector.contains("ed25519") + && txt.starts_with("v=DKIM1; k=ed25519; h=sha256; p="))) + }) +} + +fn assert_key_has_dns_record(records: &[NamedDnsRecord], domain: &str, key: &Dkim1Signature) { + assert!( + has_dns_record(records, domain, &key.selector), "No DNS record found for DKIM key with selector {}, records: {:#?}", - key.selector, records + key.selector, + records ); } @@ -396,19 +1007,11 @@ fn assert_zone_file_omits_key(zone_file: &str, key: &Dkim1Signature) { ); } -fn assert_key_has_no_dns_record(records: &[NamedDnsRecord], key: &Dkim1Signature) { - let expected = format!("{}._domainkey.dkim.org.", key.selector); - for record in records { - if record.name == expected - && let DnsRecord::TXT(txt) = &record.record - && ((key.selector.contains("rsa") && txt.starts_with("v=DKIM1; k=rsa; h=sha256; p=")) - || (key.selector.contains("ed25519") - && txt.starts_with("v=DKIM1; k=ed25519; h=sha256; p="))) - { - panic!( - "Unexpected DNS record found for DKIM key with selector {}, records: {:#?}", - key.selector, records - ); - } - } +fn assert_key_has_no_dns_record(records: &[NamedDnsRecord], domain: &str, key: &Dkim1Signature) { + assert!( + !has_dns_record(records, domain, &key.selector), + "Unexpected DNS record found for DKIM key with selector {}, records: {:#?}", + key.selector, + records + ); } diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index c26bbb7..7ea212c 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -211,6 +211,12 @@ async fn antispam() { Instant::now() + Duration::from_secs(100), ); } + // A DNSBL answer with several codes must produce one tag per code + test.server.dnsbl_add( + "dbl-multi.com.dbl.spamhaus.org", + vec!["127.0.1.2".parse().unwrap(), "127.0.1.4".parse().unwrap()], + Instant::now() + Duration::from_secs(100), + ); for mx in [ "domain.org", "domain.co.uk", diff --git a/tests/src/system/antispam.rs b/tests/src/system/antispam.rs index 99bfe04..5db9eea 100644 --- a/tests/src/system/antispam.rs +++ b/tests/src/system/antispam.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::utils::{jmap::JmapUtils, server::TestServer}; @@ -11,8 +13,8 @@ use registry::{ enums::{Permission, TaskSpamFilterMaintenanceType, TaskStoreMaintenanceType}, prelude::{ObjectType, Property}, structs::{ - Permissions, PermissionsList, SpamTrainingSample, Task, TaskSpamFilterMaintenance, - TaskStatus, TaskStoreMaintenance, + Jmap, Permissions, PermissionsList, SpamTrainingSample, Task, + TaskSpamFilterMaintenance, TaskStatus, TaskStoreMaintenance, }, }, types::map::Map, @@ -26,6 +28,21 @@ pub async fn test(test: &mut TestServer) { // Create test accounts let admin = test.account("admin@example.org"); + + // inbuxa: the quota test leaves uploads expiring after one second, at + // most three at a time. This test imports twenty samples, and a debug + // build can take longer than that between upload and import. + admin + .registry_update_setting( + Jmap::default(), + &[ + Property::UploadQuota, + Property::MaxUploadCount, + Property::UploadTtl, + ], + ) + .await; + admin.reload_settings().await; let account = test .create_user_account( "admin@example.org", @@ -209,6 +226,14 @@ pub async fn test(test: &mut TestServer) { assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 11); assert_eq!(samples.len(), 20); + // Removing the duplicate sample of a reclassified email should not remove the blob of the kept sample + for (id, sample) in &samples { + assert!( + client.download(&sample.blob_id.to_string()).await.is_ok(), + "blob of sample {id} is not accessible" + ); + } + // Adding a training sample without permissions should fail assert_eq!( account diff --git a/tests/src/utils/dns.rs b/tests/src/utils/dns.rs index 4645ccd..3325f0d 100644 --- a/tests/src/utils/dns.rs +++ b/tests/src/utils/dns.rs @@ -77,14 +77,12 @@ impl DnsCache for Server { fn dnsbl_add(&self, name: &str, value: Vec, valid_until: std::time::Instant) { self.inner.cache.dns_rbl.insert_with_expiry( name.into(), - Some(Arc::new(IpResolver::new( + Some( value - .iter() - .copied() - .next() - .unwrap_or(Ipv4Addr::BROADCAST) - .into(), - ))), + .into_iter() + .map(|ip| IpResolver::new(ip.into())) + .collect(), + ), valid_until, ); }