diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cc23c6..300f23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,39 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.16.24] - 2026-09-27 + +If you are upgrading from v0.16.x, replace the binary (or run `docker pull`). If you are upgrading from v0.15.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) for more information on how to upgrade from previous versions. + +## Added +- DNS: PowerDNS Authoritative provider for automatic DNS record management. + +## Changed + +## Fixed +- Troubleshoot tool: `TLSA` records are looked up for every MX host, including hosts whose zone is not DNSSEC signed. +- Spam filter: + - OpenPhish and PhishTank entries containing uppercase characters never match, since message URLs are lowercased while HTTP lookup entries keep their original case. HTTP lookups now match keys case-insensitively. + - URL shortener links are followed using the lowercased URL, so case-sensitive short links resolve to the wrong destination or not at all. + - Incremental training never advances its position past the first run, so every retained sample added since then is trained again, and counted again in the reservoir, on each run until it expires. + - Updating the rules only adds new objects, so upstream changes to existing rules, DNSBL servers, HTTP lookups, lookup keys and file extensions never reach an existing installation. + - Updating the rules reports success when objects fail to import, or when a configuration error stops the updated settings from being activated. +- JMAP: + - A `PushSubscription` created within the verification rate limit window of another one on the same account never receives its `PushVerification`, since the blocked verification is dropped instead of being sent once the window expires. + - A push notification retried after a failed delivery can report an older state than a change queued during the failed attempt, since the older state changes are merged last and overwrite the newer ones. + - Changes made while a push request is in flight are not delivered until the next change reaches the same subscription, since a successful delivery cancels the pending retry. + - The VAPID `aud` claim is derived from a hand-written parse of the push URL, so a crafted push URL can make the server sign a token for a push service other than the one the request is sent to. + - `Email/import` rejects a `blobId` that refers to a `Blob/upload` creation id in the same request (`"#u0"`) with `Invalid blob id.`. + - `Email/set` with a full `mailboxIds` object identical to the current mailboxes, together with a keyword change, stores the message with IMAP UID 0, so IMAP clients stop seeing it. +- MTA: + - A node without the `outboundMta` role stops replying to `DATA` and to JMAP submissions once about 1024 messages have been queued on it. + - MX records are resolved through the DNSSEC-validating resolver even when DANE is disabled. + - A `DATA` stage Sieve script does not see headers added by milters or MTA hooks, and discards every milter and MTA hook change when it edits the message. +- MySQL: Range deletions and search index removals start with a single unbounded `DELETE` and switch to chunks only after a timeout. +- IMAP: `COPY` and `MOVE` fail with `NO [CONTACTADMIN]` when another session changes the same message at the same time. +- Autodiscover: Implicit TLS ports (993, 995, 465) are advertised with `TLS`, which Outlook reads as STARTTLS. +- HTTP: Idle keep-alive connections are never closed. + ## [0.16.23] - 2026-09-21 If you are upgrading from v0.16.x, replace the binary (or run `docker pull`). If you are upgrading from v0.15.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) for more information on how to upgrade from previous versions. diff --git a/Cargo.lock b/Cargo.lock index 396def3..c6e73fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,7 +116,7 @@ dependencies = [ "once_cell", "serde", "version_check", - "zerocopy 0.8.57", + "zerocopy 0.8.59", ] [[package]] @@ -221,7 +221,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", ] @@ -318,7 +318,7 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-rustls", "tokio-stream", @@ -412,7 +412,7 @@ dependencies = [ "quick-xml 0.38.4", "rust-ini", "serde", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "url", ] @@ -447,7 +447,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "838b36c8dc927b6db1b6c6b8f5d05865f2213550b9e83bf92fa99ed6525472c0" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -740,16 +740,6 @@ dependencies = [ "wyz", ] -[[package]] -name = "bitvec-nom2" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d988fcc40055ceaa85edc55875a08f8abd29018582647fd82ad6128dba14a5f0" -dependencies = [ - "bitvec", - "nom", -] - [[package]] name = "blake2" version = "0.10.6" @@ -882,7 +872,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tokio", "tokio-stream", @@ -1110,9 +1100,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.7" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "jobserver", @@ -1302,7 +1292,7 @@ dependencies = [ [[package]] name = "common" -version = "0.16.23" +version = "0.16.24" dependencies = [ "aes-gcm-siv", "ahash", @@ -1487,7 +1477,7 @@ checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "coordinator" -version = "0.16.23" +version = "0.16.24" dependencies = [ "async-nats", "futures", @@ -1899,7 +1889,7 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "dav" -version = "0.16.23" +version = "0.16.24" dependencies = [ "calcard", "chrono", @@ -1922,7 +1912,7 @@ dependencies = [ [[package]] name = "dav-proto" -version = "0.16.23" +version = "0.16.24" dependencies = [ "calcard", "chrono", @@ -2022,7 +2012,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -2135,7 +2125,7 @@ dependencies = [ [[package]] name = "directory" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "argon2 0.6.0", @@ -2206,16 +2196,16 @@ dependencies = [ [[package]] name = "dns-update" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "887fbd45d910c4dd5808f65c0cd686d4f0f756450b9a6c7ecce4a9f450657509" +checksum = "5b55d6bab7e8f88b1ff3079da050676de0fbc0589014bb52bf39975affb8ad19" dependencies = [ "aws-lc-rs", "base64 0.23.1", - "chrono", "hex", "hickory-net", "hickory-proto", + "jiff", "quick-xml 0.42.0", "reqwest 0.13.5", "rustls", @@ -2315,7 +2305,7 @@ dependencies = [ "openssl", "serde", "sha2 0.10.9", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -2376,7 +2366,7 @@ dependencies = [ [[package]] name = "email" -version = "0.16.23" +version = "0.16.24" dependencies = [ "aes 0.9.3", "aes-gcm 0.11.1", @@ -2418,13 +2408,12 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.41" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", "core_detect", - "multiversion", "multiversion_no_op", "rustversion", "scopeguard", @@ -2485,7 +2474,7 @@ dependencies = [ [[package]] name = "event_macro" -version = "0.16.23" +version = "0.16.24" dependencies = [ "quote", "syn 3.0.6", @@ -2594,9 +2583,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "fixed_decimal" @@ -2684,7 +2673,7 @@ dependencies = [ "memchr", "mime", "serde", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -3013,7 +3002,7 @@ dependencies = [ [[package]] name = "groupware" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "calcard", @@ -3102,8 +3091,6 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", - "equivalent", "foldhash 0.2.0", ] @@ -3173,7 +3160,7 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tinyvec", "tokio", @@ -3199,7 +3186,7 @@ dependencies = [ "rand 0.10.3", "ring", "rustls-pki-types", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tinyvec", "tracing", @@ -3228,7 +3215,7 @@ dependencies = [ "rustls", "smallvec", "system-configuration", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-rustls", "tracing", @@ -3302,7 +3289,7 @@ dependencies = [ [[package]] name = "http" -version = "0.16.23" +version = "0.16.24" dependencies = [ "async-stream", "base64 0.23.1", @@ -3398,7 +3385,7 @@ dependencies = [ [[package]] name = "http_proto" -version = "0.16.23" +version = "0.16.24" dependencies = [ "common", "compact_str", @@ -3518,16 +3505,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http 1.5.0", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -3836,9 +3824,9 @@ dependencies = [ [[package]] name = "icu_time_data" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ff86fa5851a77fc0a977b72d717f777bd9230415a247219d7898aefb1517a" +checksum = "d9e21be527807cda7562cbfbdb04179f9cca46c12691d6a8073ffae6cfd7aeec" [[package]] name = "idea" @@ -3884,7 +3872,7 @@ checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" [[package]] name = "imap" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "common", @@ -3909,7 +3897,7 @@ dependencies = [ [[package]] name = "imap_proto" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "base64 0.23.1", @@ -3924,7 +3912,7 @@ dependencies = [ [[package]] name = "inbuxa" -version = "0.16.23" +version = "0.16.24" dependencies = [ "common", "coordinator", @@ -3932,7 +3920,7 @@ dependencies = [ "directory", "email", "groupware", - "http 0.16.23", + "http 0.16.24", "http_proto", "imap", "jmap", @@ -4214,7 +4202,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.16.23" +version = "0.16.24" dependencies = [ "async-stream", "base64 0.23.1", @@ -4297,7 +4285,7 @@ dependencies = [ [[package]] name = "jmap_proto" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "calcard", @@ -4326,7 +4314,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link 0.2.1", ] @@ -4375,9 +4363,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -4540,7 +4528,7 @@ dependencies = [ "percent-encoding", "rustls", "rustls-native-certs", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-rustls", "tokio-stream", @@ -4603,9 +4591,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -4684,9 +4672,9 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" dependencies = [ "hashbrown 0.17.1", ] @@ -4803,7 +4791,7 @@ dependencies = [ [[package]] name = "managesieve" -version = "0.16.23" +version = "0.16.24" dependencies = [ "common", "compact_str", @@ -4938,7 +4926,7 @@ checksum = "c797b9d6bb23aab2fc369c65f871be49214f5c759af65bde26ffaaa2b646b492" [[package]] name = "migration" -version = "0.16.23" +version = "0.16.24" dependencies = [ "common", "email", @@ -5051,27 +5039,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.6", -] - [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -5113,7 +5080,7 @@ dependencies = [ "quote", "syn 2.0.119", "termcolor", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -5138,7 +5105,7 @@ dependencies = [ "rustls", "serde", "socket2 0.6.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-rustls", "tokio-util", @@ -5170,7 +5137,7 @@ dependencies = [ "serde_json", "sha1 0.10.7", "sha2 0.10.9", - "thiserror 2.0.20", + "thiserror 2.0.21", "uuid", ] @@ -5209,7 +5176,7 @@ dependencies = [ [[package]] name = "nlp" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "hashify", @@ -5536,7 +5503,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -5565,7 +5532,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest 0.13.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tonic", "tonic-types", @@ -5600,7 +5567,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -5787,9 +5754,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" dependencies = [ "memchr", "ucd-trie", @@ -5797,9 +5764,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" dependencies = [ "pest", "pest_generator", @@ -5807,9 +5774,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" dependencies = [ "pest", "pest_meta", @@ -5820,9 +5787,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" dependencies = [ "pest", ] @@ -6041,7 +6008,7 @@ dependencies = [ [[package]] name = "pop3" -version = "0.16.23" +version = "0.16.24" dependencies = [ "common", "directory", @@ -6132,7 +6099,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.57", + "zerocopy 0.8.59", ] [[package]] @@ -6173,9 +6140,9 @@ dependencies = [ [[package]] name = "privdrop" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70722a5a3728c9603c8d9469b64b8d1ee54dae6d74e24146da7f501b4c76540f" +checksum = "454e2cdcd17f5b868a2f600b14660c1af558b85b69540b15ddc6b0a890d5b7f0" dependencies = [ "libc", "nix", @@ -6243,7 +6210,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -6280,9 +6247,9 @@ dependencies = [ [[package]] name = "proxy-header" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1493f63ddddfba840c3169e997c2905d09538ace72d64e84af6324c6e0e065" +checksum = "accf7ba1ae5e4f5b6acfb2d88d3cf68d03c333fca0260b06fb9aafcfcf22e2f5" dependencies = [ "pin-project-lite", "tokio", @@ -6290,9 +6257,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.235" +version = "2.1.238" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8319b56ff38ca0522b4e1e40bfa2b5de7f62dc89fc1e9033eac365551ec58e0e" +checksum = "2e6aa2cd4e8e062e78ac5d7df6206e2f8e39b624f658efd6f34f9aa1928ac5da" dependencies = [ "psl-types", ] @@ -6403,7 +6370,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2 0.6.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -6428,7 +6395,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "slab", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -6629,12 +6596,11 @@ dependencies = [ [[package]] name = "rasn" -version = "0.28.14" +version = "0.28.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d1b71dd951343df0fe30b11bca09518f3aba8e5bd0ece4564adbc2dabd875fa" +checksum = "9ff8f77a3a7082ed9f3e878927f34b1d633293fce9363d8ba14ee1c24bd2e967" dependencies = [ "bitvec", - "bitvec-nom2", "bytes", "cfg-if", "chrono", @@ -6652,9 +6618,9 @@ dependencies = [ [[package]] name = "rasn-cms" -version = "0.28.14" +version = "0.28.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4abf4c2fe5537b99e2bf3099a7ad26e40bd9cf51bc003600ed58dbbff69a1c" +checksum = "2d11eca85a3699d3e52795874704ddd6d2f03bda3585be3e6655a0d9a166e84e" dependencies = [ "rasn", "rasn-pkix", @@ -6662,9 +6628,9 @@ dependencies = [ [[package]] name = "rasn-derive" -version = "0.28.14" +version = "0.28.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "979eafa601d351f7f6490f1d2f4decc168e8e51cb6517c6c6ff80111951129d5" +checksum = "6fe71aa61cfb0191e50b95ebd72d454e9eeabff8a35b6252b99f2e90b7ef4c6d" dependencies = [ "proc-macro2", "rasn-derive-impl", @@ -6673,9 +6639,9 @@ dependencies = [ [[package]] name = "rasn-derive-impl" -version = "0.28.14" +version = "0.28.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee3136c97d6f2553c0d9f84d2c2555bb87ae9b365c5c9274e8bc5c366174431" +checksum = "ac6b7d207ad810df71b46ef90916dbb71fe366e0ad58a62e737fe6098525684c" dependencies = [ "either", "itertools 0.13.0", @@ -6687,9 +6653,9 @@ dependencies = [ [[package]] name = "rasn-pkix" -version = "0.28.14" +version = "0.28.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c029da7ed3b94dd93b75299431984081c1eaf88ce79771c21b056f0ebf6fb964" +checksum = "773eb443e9c58d9d356c065bbfea2c04322277c44073b4e0716967d8ccd7ef58" dependencies = [ "rasn", ] @@ -6762,9 +6728,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9" dependencies = [ "arcstr", "async-lock", @@ -6808,7 +6774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ "libredox", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -6862,7 +6828,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "registry" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "hashify", @@ -7117,7 +7083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown 0.16.1", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -7178,7 +7144,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sysinfo", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tokio", "tokio-stream", @@ -7287,9 +7253,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", @@ -7308,9 +7274,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" @@ -7419,7 +7385,7 @@ dependencies = [ [[package]] name = "scim" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "base64 0.23.1", @@ -7445,7 +7411,7 @@ dependencies = [ [[package]] name = "scim-proto" -version = "0.16.23" +version = "0.16.24" dependencies = [ "hashify", "serde", @@ -7582,7 +7548,7 @@ dependencies = [ "sha2 0.10.9", "sha3 0.10.9", "slh-dsa", - "thiserror 2.0.20", + "thiserror 2.0.21", "twofish", "typenum", "x25519-dalek", @@ -7771,7 +7737,7 @@ dependencies = [ [[package]] name = "services" -version = "0.16.23" +version = "0.16.24" dependencies = [ "aes-gcm 0.11.1", "aho-corasick", @@ -8041,15 +8007,15 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint 0.4.8", "num-traits", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", ] [[package]] name = "siphasher" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" [[package]] name = "slab" @@ -8078,13 +8044,13 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.1" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" [[package]] name = "smtp" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "base64 0.23.1", @@ -8175,7 +8141,7 @@ dependencies = [ [[package]] name = "spam-filter" -version = "0.16.23" +version = "0.16.24" dependencies = [ "common", "compact_str", @@ -8295,7 +8261,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "arc-swap", @@ -8546,7 +8512,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -8555,7 +8521,7 @@ dependencies = [ [[package]] name = "tests" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "aws-lc-rs", @@ -8577,7 +8543,7 @@ dependencies = [ "form_urlencoded", "futures", "groupware", - "http 0.16.23", + "http 0.16.24", "http_proto", "hyper", "hyper-util", @@ -8639,11 +8605,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -8659,9 +8625,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", @@ -8762,7 +8728,7 @@ checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530" dependencies = [ "futures-util", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-rustls", ] @@ -9149,7 +9115,7 @@ dependencies = [ [[package]] name = "trc" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "base64 0.23.1", @@ -9194,7 +9160,7 @@ dependencies = [ "rustls", "rustls-pki-types", "sha1 0.10.7", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -9210,7 +9176,7 @@ dependencies = [ "log", "rand 0.10.3", "sha1 0.11.0", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -9258,7 +9224,7 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "types" -version = "0.16.23" +version = "0.16.24" dependencies = [ "blake3", "compact_str", @@ -9427,7 +9393,7 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utils" -version = "0.16.23" +version = "0.16.24" dependencies = [ "ahash", "arcstr", @@ -9589,9 +9555,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -9602,19 +9568,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9622,9 +9589,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", @@ -9635,9 +9602,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -9670,9 +9637,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -10074,7 +10041,7 @@ dependencies = [ "oid-registry", "ring", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", ] @@ -10628,11 +10595,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" dependencies = [ - "zerocopy-derive 0.8.57", + "zerocopy-derive 0.8.59", ] [[package]] @@ -10648,9 +10615,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" dependencies = [ "proc-macro2", "quote", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 37bd7d1..e305a13 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "common" -version = "0.16.23" +version = "0.16.24" edition = "2024" build = "build.rs" diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index 5f759b4..7a5e964 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -570,8 +570,8 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { // inbuxa: rules are always to hand, since a copy ships with the server // (spam_rules). They load on first boot, and again when the bundled - // version differs from the one last loaded, which only adds what's - // missing: new tags and rules, never a changed score. + // rules differ from the ones last loaded: new tags and rules, fixes to + // rules nobody edited, never a changed score or an admin's edit. let rules_url = super::spam_rules::rules_url( bp.registry .object::(Id::singleton()) @@ -582,7 +582,7 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { && super::spam_rules::applied_version(&bp.data_store) .await? .as_deref() - != Some(super::spam_rules::BUNDLED_SPAM_RULES_VERSION); + != Some(super::spam_rules::BUNDLED_SPAM_RULES_APPLIED); if bp.registry.count_object(ObjectType::SpamRule).await? == 0 || bundled_is_new { let mut batch = BatchBuilder::new(); batch.schedule_task(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance { diff --git a/crates/common/src/manager/spam_rules.rs b/crates/common/src/manager/spam_rules.rs index acac5e2..050540c 100644 --- a/crates/common/src/manager/spam_rules.rs +++ b/crates/common/src/manager/spam_rules.rs @@ -12,11 +12,16 @@ //! and license) and uses it whenever no other source is configured. The rules //! URL remains an operator override (`https://` or `file://`). //! -//! Loading rules only ever adds what's missing, never changes an existing rule -//! or score. They load on first boot, and again whenever the bundled version -//! differs from the one last applied, so an upgrade brings new tags (the AI -//! classifier's `LLM_*` scores, say) to an install that already had rules. +//! Loading rules adds what's missing and brings an existing rule up to date, +//! but never touches one an admin edited: every object an update writes is +//! fingerprinted, and one that no longer matches its fingerprint is kept as +//! it is. Tags (scores) are never replaced. Switching a rule on or off isn't +//! an edit, and is kept either way. They load on first boot, and again +//! whenever the bundled rules differ from the ones last applied, so an +//! upgrade brings new tags (the AI classifier's `LLM_*` scores, say) and +//! fixed rules to an install that already had rules. +use registry::{schema::prelude::ObjectType, types::EnumImpl}; use std::io::Read; use store::{ SUBSPACE_INBUXA, Store, ValueKey, @@ -27,13 +32,17 @@ use trc::AddContext; /// The version of spam-filter the embedded rules come from. pub const BUNDLED_SPAM_RULES_VERSION: &str = "3.0.2"; +/// What's recorded once the bundled rules are loaded: their version, then the +/// fork's own generation of the update, so a change to how an update applies +/// runs it once more. Generation 2 fingerprints (upstream v0.16.24). +pub const BUNDLED_SPAM_RULES_APPLIED: &str = "3.0.2+2"; + static BUNDLED_SPAM_RULES: &[u8] = include_bytes!("../../../../resources/spam-filter/spam-filter-rules.json.gz"); /// Upstream's default rules source, the value every install created before /// the rules were bundled has saved. Read only to treat it as unset. -const LEGACY_DEFAULT_URL: &str = - "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz"; +const LEGACY_DEFAULT_URL: &str = "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz"; /// The URL to fetch rules from, or `None` for the bundled rules. An empty /// setting and upstream's old default both mean the bundled rules. @@ -57,14 +66,49 @@ fn applied_key() -> ValueClass { }) } -/// The bundled version last loaded into the registry, if any. +fn fingerprint_key(object: ObjectType, id: u64) -> ValueClass { + let mut key = b"Sf".to_vec(); + key.extend_from_slice(object.as_str().as_bytes()); + key.push(0); + key.extend_from_slice(&id.to_be_bytes()); + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key, + }) +} + +/// The fingerprint of what a rules update last wrote to this object, if one +/// did. +pub async fn fingerprint(data: &Store, object: ObjectType, id: u64) -> trc::Result> { + data.get_value::(ValueKey::from(fingerprint_key(object, id))) + .await + .caused_by(trc::location!()) +} + +/// Records the fingerprint of what a rules update wrote to this object. +pub async fn set_fingerprint( + data: &Store, + object: ObjectType, + id: u64, + fingerprint: &str, +) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + batch.set(fingerprint_key(object, id), fingerprint.as_bytes().to_vec()); + data.write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + +/// The bundled rules last loaded into the registry, if any +/// ([`BUNDLED_SPAM_RULES_APPLIED`]'s form). pub async fn applied_version(data: &Store) -> trc::Result> { data.get_value::(ValueKey::from(applied_key())) .await .caused_by(trc::location!()) } -/// Records that the bundled rules of this version have been loaded. +/// Records that the bundled rules have been loaded. pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()> { let mut batch = BatchBuilder::new(); batch.set(applied_key(), version.as_bytes().to_vec()); @@ -90,6 +134,15 @@ mod tests { ); } + #[test] + fn applied_marker_names_the_bundled_version() { + assert!( + BUNDLED_SPAM_RULES_APPLIED + .strip_prefix(BUNDLED_SPAM_RULES_VERSION) + .is_some_and(|generation| generation.starts_with('+')) + ); + } + #[test] fn bundled_rules_parse_and_score_the_ai_tags() { let rules: serde_json::Value = serde_json::from_slice(&bundled_rules().unwrap()).unwrap(); diff --git a/crates/common/src/network/autoconfig/autodiscover.rs b/crates/common/src/network/autoconfig/autodiscover.rs index cefea55..eb17351 100644 --- a/crates/common/src/network/autoconfig/autodiscover.rs +++ b/crates/common/src/network/autoconfig/autodiscover.rs @@ -10,8 +10,9 @@ use crate::{Server, manager::application::Resource}; use quick_xml::Reader; use quick_xml::XmlVersion; use quick_xml::events::Event; -use registry::schema::enums::ServiceProtocol; +use registry::schema::{enums::ServiceProtocol, structs::Service}; use std::fmt::Write; +use utils::map::vec_map::VecMap; impl Server { pub async fn handle_autodiscover_request( @@ -26,89 +27,103 @@ impl Server { .details("Failed to parse autodiscover request") .ctx(trc::Key::Reason, err) })?; - let default_host = &self.core.network.server_name; - - // Build XML response - let mut config = String::with_capacity(1024); - let _ = writeln!(&mut config, ""); - let _ = writeln!( - &mut config, - "" - ); - let _ = writeln!( - &mut config, - "\t" - ); - let _ = writeln!(&mut config, "\t\t"); - let _ = writeln!( - &mut config, - "\t\t\t{emailaddress}" - ); - let _ = writeln!( - &mut config, - "\t\t\t{emailaddress}" - ); - // DeploymentId is a required field of User but we are not a MS Exchange server so use a random value - let _ = writeln!( - &mut config, - "\t\t\t644560b8-a1ce-429c-8ace-23395843f701" - ); - let _ = writeln!(&mut config, "\t\t"); - let _ = writeln!(&mut config, "\t\t"); - let _ = writeln!(&mut config, "\t\t\temail"); - let _ = writeln!(&mut config, "\t\t\tsettings"); // inbuxa: legacy-protocols LP-7, LP-14a let legacy_off = match emailaddress.rsplit_once('@') { Some((_, domain)) => self.legacy_off_for(domain).await?, None => self.legacy_off_for("").await?, }; - for (protocol, service) in &self.core.network.info.services { - if legacy_off.service(protocol) { - continue; - } - let (protocol, ports) = match protocol { - ServiceProtocol::Imap => ("IMAP", [143, 993]), - ServiceProtocol::Pop3 => ("POP3", [110, 995]), - ServiceProtocol::Smtp => ("SMTP", [587, 465]), - _ => continue, - }; - - for (is_tls, port) in ports.into_iter().enumerate() { - if is_tls == 1 || service.cleartext { - let server_name = service.hostname.as_deref().unwrap_or(default_host); - let _ = writeln!(&mut config, "\t\t\t"); - let _ = writeln!(&mut config, "\t\t\t\t{protocol}",); - let _ = writeln!(&mut config, "\t\t\t\t{server_name}"); - let _ = writeln!(&mut config, "\t\t\t\t{port}"); - let _ = writeln!(&mut config, "\t\t\t\t{emailaddress}"); - let _ = writeln!(&mut config, "\t\t\t\ton"); - let _ = writeln!(&mut config, "\t\t\t\t0"); - let _ = writeln!(&mut config, "\t\t\t\t0"); - let _ = writeln!( - &mut config, - "\t\t\t\t{}", - if is_tls == 1 { "on" } else { "off" } - ); - if is_tls == 1 { - let _ = writeln!(&mut config, "\t\t\t\tTLS"); - } - let _ = writeln!(&mut config, "\t\t\t\toff"); - let _ = writeln!(&mut config, "\t\t\t"); - } - } - } - - let _ = writeln!(&mut config, "\t\t"); - let _ = writeln!(&mut config, "\t"); - let _ = writeln!(&mut config, ""); Ok(Resource::new( "application/xml; charset=utf-8", - config.into_bytes(), + build_autodiscover_response( + &emailaddress, + &self.core.network.server_name, + &self.core.network.info.services, + |protocol| legacy_off.service(protocol), + ) + .into_bytes(), )) } } +fn build_autodiscover_response( + emailaddress: &str, + default_host: &str, + services: &VecMap, + switched_off: impl Fn(&ServiceProtocol) -> bool, +) -> String { + // Build XML response + let mut config = String::with_capacity(1024); + let _ = writeln!(&mut config, ""); + let _ = writeln!( + &mut config, + "" + ); + let _ = writeln!( + &mut config, + "\t" + ); + let _ = writeln!(&mut config, "\t\t"); + let _ = writeln!( + &mut config, + "\t\t\t{emailaddress}" + ); + let _ = writeln!( + &mut config, + "\t\t\t{emailaddress}" + ); + // DeploymentId is a required field of User but we are not a MS Exchange server so use a random value + let _ = writeln!( + &mut config, + "\t\t\t644560b8-a1ce-429c-8ace-23395843f701" + ); + let _ = writeln!(&mut config, "\t\t"); + let _ = writeln!(&mut config, "\t\t"); + let _ = writeln!(&mut config, "\t\t\temail"); + let _ = writeln!(&mut config, "\t\t\tsettings"); + for (protocol, service) in services { + if switched_off(protocol) { + continue; + } + let (protocol, ports) = match protocol { + ServiceProtocol::Imap => ("IMAP", [(993, true), (143, false)]), + ServiceProtocol::Pop3 => ("POP3", [(995, true), (110, false)]), + ServiceProtocol::Smtp => ("SMTP", [(465, true), (587, false)]), + _ => continue, + }; + + // Implicit TLS is listed first so that it is preferred (RFC 8314) + for (port, is_tls) in ports { + if is_tls || service.cleartext { + let server_name = service.hostname.as_deref().unwrap_or(default_host); + let _ = writeln!(&mut config, "\t\t\t"); + let _ = writeln!(&mut config, "\t\t\t\t{protocol}",); + let _ = writeln!(&mut config, "\t\t\t\t{server_name}"); + let _ = writeln!(&mut config, "\t\t\t\t{port}"); + let _ = writeln!(&mut config, "\t\t\t\t{emailaddress}"); + let _ = writeln!(&mut config, "\t\t\t\ton"); + let _ = writeln!(&mut config, "\t\t\t\t0"); + let _ = writeln!(&mut config, "\t\t\t\t0"); + let (ssl, encryption) = if is_tls { + ("on", "SSL") + } else { + ("off", "TLS") + }; + let _ = writeln!(&mut config, "\t\t\t\t{ssl}"); + let _ = writeln!(&mut config, "\t\t\t\t{encryption}"); + let _ = writeln!(&mut config, "\t\t\t\toff"); + let _ = writeln!(&mut config, "\t\t\t"); + } + } + } + + let _ = writeln!(&mut config, "\t\t"); + let _ = writeln!(&mut config, "\t"); + let _ = writeln!(&mut config, ""); + + config +} + fn parse_autodiscover_request(bytes: &[u8]) -> Result { if bytes.is_empty() { return Err("Empty request body".to_string()); @@ -211,4 +226,79 @@ mod tests { "email@example.com" ); } + + #[test] + fn autodiscover_encryption() { + use registry::schema::{enums::ServiceProtocol, structs::Service}; + use utils::map::vec_map::VecMap; + + fn tag<'x>(block: &'x str, name: &str) -> &'x str { + block + .split_once(&format!("<{name}>")) + .and_then(|(_, rest)| rest.split_once(&format!(""))) + .map(|(value, _)| value) + .unwrap() + } + + for (cleartext, expected) in [ + ( + false, + vec![ + ("IMAP", "993", "on", "SSL"), + ("POP3", "995", "on", "SSL"), + ("SMTP", "465", "on", "SSL"), + ], + ), + ( + true, + vec![ + ("IMAP", "993", "on", "SSL"), + ("IMAP", "143", "off", "TLS"), + ("POP3", "995", "on", "SSL"), + ("POP3", "110", "off", "TLS"), + ("SMTP", "465", "on", "SSL"), + ("SMTP", "587", "off", "TLS"), + ], + ), + ] { + let services: VecMap = [ + ServiceProtocol::Imap, + ServiceProtocol::Pop3, + ServiceProtocol::Smtp, + ServiceProtocol::Jmap, + ] + .into_iter() + .map(|protocol| { + ( + protocol, + Service { + hostname: None, + cleartext, + }, + ) + }) + .collect(); + let response = super::build_autodiscover_response( + "user@example.com", + "mail.example.com", + &services, + |_| false, + ); + + assert_eq!( + response + .split("") + .skip(1) + .map(|block| ( + tag(block, "Type"), + tag(block, "Port"), + tag(block, "SSL"), + tag(block, "Encryption"), + )) + .collect::>(), + expected, + "cleartext: {cleartext}" + ); + } + } } diff --git a/crates/common/src/network/dns/update.rs b/crates/common/src/network/dns/update.rs index 0d88d52..76faa02 100644 --- a/crates/common/src/network/dns/update.rs +++ b/crates/common/src/network/dns/update.rs @@ -961,6 +961,20 @@ impl DnsUpdater { ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), + DnsServer::PowerDns(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_pdns( + server.api_key.secret().await?, + server.endpoint, + server.server_id, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), DnsServer::Safedns(server) => Ok(DnsUpdater { polling_interval: server.polling_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), diff --git a/crates/common/src/network/webpush.rs b/crates/common/src/network/webpush.rs index 1f8bf65..55cfa09 100644 --- a/crates/common/src/network/webpush.rs +++ b/crates/common/src/network/webpush.rs @@ -6,33 +6,79 @@ * Modified by Coffey Labs in 2026 for INBUXA. */ +use ahash::AHashMap; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use p256::{ SecretKey, ecdsa::{Signature, SigningKey, signature::Signer}, pkcs8::{DecodePrivateKey, PrivateKeyInfo, der::SecretDocument}, }; +use parking_lot::Mutex; +use reqwest::{Url, header::HeaderValue}; +use std::sync::Arc; const VAPID_TOKEN_TTL: u64 = 12 * 60 * 60; +const VAPID_TOKEN_REFRESH: u64 = VAPID_TOKEN_TTL / 2; #[derive(Clone)] pub struct Vapid { key: VapidKey, contact: Option, + tokens: Arc>>, +} + +struct VapidToken { + authorization: HeaderValue, + issued_at: u64, } impl Vapid { pub fn new(key: VapidKey, contact: Option) -> Self { - Self { key, contact } + Self { + key, + contact, + tokens: Arc::default(), + } } pub fn public_key(&self) -> &str { self.key.public_key() } - pub fn authorization(&self, endpoint: &str, now: u64) -> Option { - self.key - .authorization(endpoint, self.contact.as_deref(), now) + pub fn authorization(&self, endpoint: &str, now: u64) -> Option { + let prefix = endpoint_prefix(endpoint)?; + if let Some(token) = self + .tokens + .lock() + .get(prefix) + .filter(|token| token.is_fresh(now)) + { + return Some(token.authorization.clone()); + } + + let authorization = HeaderValue::try_from(self.key.authorization( + endpoint, + self.contact.as_deref(), + now, + )?) + .ok()?; + let mut tokens = self.tokens.lock(); + tokens.retain(|_, token| token.is_fresh(now)); + tokens.insert( + prefix.to_string(), + VapidToken { + authorization: authorization.clone(), + issued_at: now, + }, + ); + Some(authorization) + } +} + +impl VapidToken { + fn is_fresh(&self, now: u64) -> bool { + now.checked_sub(self.issued_at) + .is_some_and(|age| age < VAPID_TOKEN_REFRESH) } } @@ -105,41 +151,15 @@ impl VapidKey { } } -fn endpoint_origin(url: &str) -> Option { +fn endpoint_prefix(url: &str) -> Option<&str> { let (scheme, rest) = url.split_once("://")?; - let scheme = scheme.to_ascii_lowercase(); let authority = rest.split(['/', '?', '#']).next()?; - let authority = authority - .rsplit_once('@') - .map(|(_, host)| host) - .unwrap_or(authority); - if authority.is_empty() { - return None; - } + url.get(..scheme.len() + "://".len() + authority.len()) +} - let (host, port) = if let Some(rest) = authority.strip_prefix('[') { - let (addr, tail) = rest.split_once(']')?; - ( - format!("[{}]", addr.to_ascii_lowercase()), - tail.strip_prefix(':').filter(|port| !port.is_empty()), - ) - } else if let Some((host, port)) = authority.rsplit_once(':') { - ( - host.to_ascii_lowercase(), - Some(port).filter(|p| !p.is_empty()), - ) - } else { - (authority.to_ascii_lowercase(), None) - }; - - match port { - Some(port) - if !((scheme == "https" && port == "443") || (scheme == "http" && port == "80")) => - { - Some(format!("{scheme}://{host}:{port}")) - } - _ => Some(format!("{scheme}://{host}")), - } +fn endpoint_origin(url: &str) -> Option { + let origin = Url::parse(url).ok()?.origin(); + origin.is_tuple().then(|| origin.ascii_serialization()) } pub fn normalize_contact(contact: &str) -> Option { @@ -206,7 +226,12 @@ mod tests { endpoint_origin("http://[2001:DB8::1]:80/p").unwrap(), "http://[2001:db8::1]" ); + assert_eq!( + endpoint_origin("https://attacker.example\\@fcm.googleapis.com/fcm/send/x").unwrap(), + "https://attacker.example" + ); assert!(endpoint_origin("not-a-url").is_none()); + assert!(endpoint_origin("mailto:admin@example.org").is_none()); } #[test] @@ -336,6 +361,47 @@ B4yDfR2rGOd2H6Kv3fQNHPj9Nu5Tks8QYMLzrX8ONCNoFnNUQl9S0r0QS6phVqD0 } } + #[test] + fn authorization_is_reused_per_endpoint_prefix() { + let vapid = Vapid::new(test_key(), None); + let now = 1_700_000_000; + let token = vapid + .authorization("https://push.example.com/push/a", now) + .unwrap(); + + assert_eq!( + vapid + .authorization("https://push.example.com/push/b?x=1", now + 60) + .unwrap(), + token + ); + assert_ne!( + vapid + .authorization("https://other.example.com/push/a", now) + .unwrap(), + token + ); + assert_ne!( + vapid + .authorization("https://push.example.com/push/a", now - 1) + .unwrap(), + token + ); + let refreshed = vapid + .authorization("https://push.example.com/push/a", now + VAPID_TOKEN_REFRESH) + .unwrap(); + assert_ne!(refreshed, token); + assert_eq!( + vapid + .authorization( + "https://push.example.com/push/c", + now + VAPID_TOKEN_REFRESH + 1 + ) + .unwrap(), + refreshed + ); + } + #[test] fn authorization_omits_subject_when_no_contact() { let key = test_key(); diff --git a/crates/coordinator/Cargo.toml b/crates/coordinator/Cargo.toml index 2a88b21..0db24a7 100644 --- a/crates/coordinator/Cargo.toml +++ b/crates/coordinator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "coordinator" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/dav-proto/Cargo.toml b/crates/dav-proto/Cargo.toml index 469a56a..b4d63b8 100644 --- a/crates/dav-proto/Cargo.toml +++ b/crates/dav-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dav-proto" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/dav/Cargo.toml b/crates/dav/Cargo.toml index 6c7879c..58a3873 100644 --- a/crates/dav/Cargo.toml +++ b/crates/dav/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dav" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 1352552..063479d 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "directory" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml index 12cc2ae..40ce0cc 100644 --- a/crates/email/Cargo.toml +++ b/crates/email/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "email" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/groupware/Cargo.toml b/crates/groupware/Cargo.toml index 3b4cf57..a1daf0e 100644 --- a/crates/groupware/Cargo.toml +++ b/crates/groupware/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "groupware" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/http-proto/Cargo.toml b/crates/http-proto/Cargo.toml index 51bb7f1..3110362 100644 --- a/crates/http-proto/Cargo.toml +++ b/crates/http-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "http_proto" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index ddf9497..99578d2 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "http" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/http/src/api/diagnose.rs b/crates/http/src/api/diagnose.rs index e6c757f..55fa81e 100644 --- a/crates/http/src/api/diagnose.rs +++ b/crates/http/src/api/diagnose.rs @@ -12,7 +12,7 @@ use common::{ }, }; use hyper::body::{Bytes, Frame}; -use mail_auth::{IpLookupStrategy, mta_sts::TlsRpt}; +use mail_auth::{DnssecStatus, IpLookupStrategy, mta_sts::TlsRpt}; use serde::{Deserialize, Serialize}; use smtp::outbound::{ client::{SmtpClient, StartTlsResult}, @@ -382,81 +382,25 @@ async fn delivery_diagnose( } } - // Fetch TLSA record - tx.send(DeliveryStage::TlsaLookupStart).await?; - - let now = Instant::now(); - let dane_policy = match server.tlsa_lookup(format!("_25._tcp.{hostname}.")).await { - Ok(TlsaResult::Secure(tlsa)) if tlsa.has_end_entities => { - tx.send(DeliveryStage::TlsaLookupSuccess { - record: tlsa.as_ref().clone(), - elapsed: now.elapsed_ms(), - }) - .await?; - - Some(tlsa) - } - Ok(TlsaResult::Secure(_)) => { - tx.send(DeliveryStage::TlsaLookupError { - elapsed: now.elapsed_ms(), - reason: "TLSA record does not have end entities".to_string(), - }) - .await?; - - None - } - Ok(TlsaResult::Bogus) => { - tx.send(DeliveryStage::TlsaLookupError { - elapsed: now.elapsed_ms(), - reason: "Bogus TLSA record".to_string(), - }) - .await?; - - continue 'outer; - } - Ok(TlsaResult::Missing) => { - tx.send(DeliveryStage::TlsaNotFound { - elapsed: now.elapsed_ms(), - reason: "No TLSA DNSSEC records found".to_string(), - }) - .await?; - - None - } - Err(err) => { - if matches!( - &err, - mail_auth::Error::Dns(mail_auth::DnsError::RecordNotFound(_)) - ) { - tx.send(DeliveryStage::TlsaNotFound { - elapsed: now.elapsed_ms(), - reason: "No TLSA records found for MX".to_string(), - }) - .await?; - - None - } else { - tx.send(DeliveryStage::TlsaLookupError { - elapsed: now.elapsed_ms(), - reason: err.to_string(), - }) - .await?; - - continue 'outer; - } - } - }; - tx.send(DeliveryStage::IpLookupStart).await?; let now = Instant::now(); - let remote_ips = match host.fqdn_hostname() { + let validate_addresses = server.core.smtp.resolvers.dnssec_available + && host.dnssec_status() == DnssecStatus::Secure; + let (remote_ips, addresses_dnssec_status) = match host.fqdn_hostname() { HostOrIp::Host(hostname) => { match server - .ip_lookup(&hostname, IpLookupStrategy::Ipv4thenIpv6, usize::MAX, false) + .ip_lookup( + &hostname, + IpLookupStrategy::Ipv4thenIpv6, + usize::MAX, + validate_addresses, + ) .await { - Ok((remote_ips, _)) if !remote_ips.is_empty() => remote_ips, + Ok((remote_ips, dnssec_status)) if !remote_ips.is_empty() => { + (remote_ips, dnssec_status) + } Ok(_) => { tx.send(DeliveryStage::IpLookupError { reason: "No IP addresses found for host".to_string(), @@ -475,7 +419,7 @@ async fn delivery_diagnose( } } } - HostOrIp::Ip(ip) => vec![ip], + HostOrIp::Ip(ip) => (vec![ip], DnssecStatus::Indeterminate), }; tx.send(DeliveryStage::IpLookupSuccess { @@ -484,6 +428,95 @@ async fn delivery_diagnose( }) .await?; + // Fetch TLSA record + tx.send(DeliveryStage::TlsaLookupStart).await?; + + let now = Instant::now(); + let dane_policy = match host.dane_status(addresses_dnssec_status) { + (DnssecStatus::Secure, _) => { + match server.tlsa_lookup(format!("_25._tcp.{hostname}.")).await { + Ok(TlsaResult::Secure(tlsa)) if tlsa.has_end_entities => { + tx.send(DeliveryStage::TlsaLookupSuccess { + record: tlsa.as_ref().clone(), + elapsed: now.elapsed_ms(), + }) + .await?; + + Some(tlsa) + } + Ok(TlsaResult::Secure(_)) => { + tx.send(DeliveryStage::TlsaLookupError { + elapsed: now.elapsed_ms(), + reason: "TLSA record does not have end entities".to_string(), + }) + .await?; + + None + } + Ok(TlsaResult::Bogus) => { + tx.send(DeliveryStage::TlsaLookupError { + elapsed: now.elapsed_ms(), + reason: "Bogus TLSA record".to_string(), + }) + .await?; + + continue 'outer; + } + Ok(TlsaResult::Missing) => { + tx.send(DeliveryStage::TlsaNotFound { + elapsed: now.elapsed_ms(), + reason: "No TLSA DNSSEC records found".to_string(), + }) + .await?; + + None + } + Err(err) => { + if matches!( + &err, + mail_auth::Error::Dns(mail_auth::DnsError::RecordNotFound(_)) + ) { + tx.send(DeliveryStage::TlsaNotFound { + elapsed: now.elapsed_ms(), + reason: "No TLSA records found for MX".to_string(), + }) + .await?; + + None + } else { + tx.send(DeliveryStage::TlsaLookupError { + elapsed: now.elapsed_ms(), + reason: err.to_string(), + }) + .await?; + + continue 'outer; + } + } + } + } + (DnssecStatus::Bogus, dnssec_entity) => { + tx.send(DeliveryStage::TlsaLookupError { + elapsed: now.elapsed_ms(), + reason: format!("Bogus {dnssec_entity} records were found"), + }) + .await?; + + continue 'outer; + } + (_, dnssec_entity) => { + tx.send(DeliveryStage::TlsaNotFound { + elapsed: now.elapsed_ms(), + reason: format!( + "{dnssec_entity} records are not DNSSEC signed, DANE does not apply" + ), + }) + .await?; + + None + } + }; + for remote_ip in remote_ips { // Start connection tx.send(DeliveryStage::ConnectionStart { remote_ip }) diff --git a/crates/http/src/request.rs b/crates/http/src/request.rs index a09f42c..c3f3c9a 100644 --- a/crates/http/src/request.rs +++ b/crates/http/src/request.rs @@ -36,7 +36,7 @@ use hyper::{ server::conn::http1, service::service_fn, }; -use hyper_util::rt::TokioIo; +use hyper_util::rt::{TokioIo, TokioTimer}; use jmap::{ api::{ ToJmapHttpResponse, event_source::EventSourceHandler, request::RequestHandler, @@ -690,6 +690,7 @@ async fn handle_session(inner: Arc, session: SessionDat let is_tls = session.stream.is_tls(); if let Err(http_err) = http1::Builder::new() + .timer(TokioTimer::new()) .keep_alive(true) .serve_connection( TokioIo::new(session.stream), @@ -875,6 +876,7 @@ async fn handle_session(inner: Arc, session: SessionDat ) .with_upgrades() .await + && !http_err.is_timeout() { if http_err.is_parse() { let server = inner.build_server(); diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index 363a4d9..f06702c 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap_proto" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 9916808..cc638c3 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 7cfd562..6a13bff 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -9,6 +9,7 @@ use crate::{ core::{MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; +use ahash::AHashMap; use common::{ipc::PushNotification, network::SessionStream, storage::index::ObjectIndexBuilder}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, @@ -22,8 +23,13 @@ use email::{ use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::copy_move::Arguments, receiver::Request, }; +use rand::RngExt; use registry::schema::enums::Permission; -use std::{sync::Arc, time::Instant}; +use std::{ + ops::RangeInclusive, + sync::Arc, + time::{Duration, Instant}, +}; use store::{ ValueKey, roaring::RoaringBitmap, @@ -36,6 +42,9 @@ use types::{ type_state::{DataType, StateChange}, }; +const MAX_MOVE_RETRIES: u32 = 3; +const MOVE_RETRY_BACKOFF_MS: RangeInclusive = 1..=15; + impl Session { pub async fn handle_copy_move( &mut self, @@ -236,148 +245,180 @@ impl SessionData { // Mailboxes are in the same account let account_id = src_mailbox.id.account_id; let dest_mailbox_id = UidMailbox::new_unassigned(dest_mailbox_id); - let mut batch = BatchBuilder::new(); + let mut written_uids = AHashMap::with_capacity(ids.len()); + let mut retries = 0; - for (id, imap_id) in ids { - // Obtain mailbox tags - let data_ = if let Some(result) = self - .get_message_data(account_id, id) - .await - .imap_ctx(&arguments.tag, trc::location!())? - { - result - } else { - continue; - }; + loop { + let mut batch = BatchBuilder::new(); + copied_ids.clear(); + did_move = false; - // Deserialize - let data = data_ - .to_unarchived::() - .imap_ctx(&arguments.tag, trc::location!())?; + for (&id, imap_id) in &ids { + // Obtain mailbox tags + let data_ = if let Some(result) = self + .get_message_data(account_id, id) + .await + .imap_ctx(&arguments.tag, trc::location!())? + { + result + } else { + continue; + }; - // Make sure the message still belongs to this mailbox - if !data - .inner - .mailboxes - .iter() - .any(|mailbox| mailbox.mailbox_id == src_mailbox.id.mailbox_id) - { - continue; - } + // Deserialize + let data = data_ + .to_unarchived::() + .imap_ctx(&arguments.tag, trc::location!())?; - // If the message is already in the destination mailbox, skip it. - if let Some(mailbox) = data - .inner - .mailboxes - .iter() - .find(|mailbox| mailbox.mailbox_id == dest_mailbox_id.mailbox_id) - { - copied_ids.push((imap_id.uid, mailbox.uid.to_native())); - - if is_move { - let mut new_data = data.inner.to_builder(); - new_data.remove_mailbox(src_mailbox.id.mailbox_id); - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .with_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), - ) - .imap_ctx(&arguments.tag, trc::location!())? - .log_vanished_item( - VanishedCollection::Email, - (src_mailbox.id.mailbox_id, imap_id.uid), - ) - .commit_point(); - did_move = true; + // Make sure the message still belongs to this mailbox + if !data + .inner + .mailboxes + .iter() + .any(|mailbox| mailbox.mailbox_id == src_mailbox.id.mailbox_id) + { + // Moved by a chunk of a previous attempt + if let Some(&uid) = written_uids.get(&id) + && data.inner.message_uid(dest_mailbox_id.mailbox_id) == Some(uid) + { + copied_ids.push((imap_id.uid, uid)); + did_move = true; + } + continue; } - continue; - } + // If the message is already in the destination mailbox, skip it. + if let Some(mailbox) = data + .inner + .mailboxes + .iter() + .find(|mailbox| mailbox.mailbox_id == dest_mailbox_id.mailbox_id) + { + let uid = mailbox.uid.to_native(); + copied_ids.push((imap_id.uid, uid)); - // Prepare changes - let mut new_data = data.inner.to_builder(); + if is_move { + let mut new_data = data.inner.to_builder(); + new_data.remove_mailbox(src_mailbox.id.mailbox_id); + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(id) + .custom( + ObjectIndexBuilder::new() + .with_current(data) + .with_changes(new_data.seal()), + ) + .imap_ctx(&arguments.tag, trc::location!())? + .log_vanished_item( + VanishedCollection::Email, + (src_mailbox.id.mailbox_id, imap_id.uid), + ) + .commit_point(); + written_uids.insert(id, uid); + did_move = true; + } - // Add destination folder - new_data.add_mailbox(dest_mailbox_id); - if is_move { - new_data.remove_mailbox(src_mailbox.id.mailbox_id); - } + continue; + } - // Assign IMAP UIDs - let ids = self - .server - .assign_email_ids( - account_id, - new_data - .mailboxes - .iter() - .filter(|m| m.uid == 0) - .map(|m| m.mailbox_id), - false, - ) - .await - .caused_by(trc::location!())?; + // Prepare changes + let mut new_data = data.inner.to_builder(); - for (uid_mailbox, uid) in new_data - .mailboxes - .iter_mut() - .filter(|m| m.uid == 0) - .zip(ids) - { - copied_ids.push((imap_id.uid, uid)); - uid_mailbox.uid = uid; - } + // Add destination folder + new_data.add_mailbox(dest_mailbox_id); + if is_move { + new_data.remove_mailbox(src_mailbox.id.mailbox_id); + } - // Prepare write batch - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .with_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), - ) - .imap_ctx(&arguments.tag, trc::location!())?; - if is_move { - batch.log_vanished_item( - VanishedCollection::Email, - (src_mailbox.id.mailbox_id, imap_id.uid), - ); - } - - // Add message to training queue - if dest_mailbox_id.mailbox_id == JUNK_ID { - self.server - .add_account_spam_sample(&mut batch, account_id, id, true, self.session_id) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - } else if src_mailbox.id.mailbox_id == JUNK_ID - && dest_mailbox_id.mailbox_id != TRASH_ID - { - self.server - .add_account_spam_sample(&mut batch, account_id, id, false, self.session_id) + // Assign IMAP UIDs + let ids = self + .server + .assign_email_ids( + account_id, + new_data + .mailboxes + .iter() + .filter(|m| m.uid == 0) + .map(|m| m.mailbox_id), + false, + ) .await + .caused_by(trc::location!())?; + + for (uid_mailbox, uid) in new_data + .mailboxes + .iter_mut() + .filter(|m| m.uid == 0) + .zip(ids) + { + copied_ids.push((imap_id.uid, uid)); + written_uids.insert(id, uid); + uid_mailbox.uid = uid; + } + + // Prepare write batch + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(id) + .custom( + ObjectIndexBuilder::new() + .with_current(data) + .with_changes(new_data.seal()), + ) .imap_ctx(&arguments.tag, trc::location!())?; + if is_move { + batch.log_vanished_item( + VanishedCollection::Email, + (src_mailbox.id.mailbox_id, imap_id.uid), + ); + } + + // Add message to training queue + if dest_mailbox_id.mailbox_id == JUNK_ID { + self.server + .add_account_spam_sample( + &mut batch, + account_id, + id, + true, + self.session_id, + ) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + } else if src_mailbox.id.mailbox_id == JUNK_ID + && dest_mailbox_id.mailbox_id != TRASH_ID + { + self.server + .add_account_spam_sample( + &mut batch, + account_id, + id, + false, + self.session_id, + ) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + } + + batch.commit_point(); + + // Update changelog + if is_move { + did_move = true; + } } - batch.commit_point(); - - // Update changelog - if is_move { - did_move = true; + // Write changes + match self.server.commit_batch(batch).await { + Ok(_) => break, + Err(err) => { + retry_after_conflict(err, &mut retries, &arguments.tag, trc::location!()) + .await? + } } } - - // Write changes - self.server - .commit_batch(batch) - .await - .imap_ctx(&arguments.tag, trc::location!())?; } else { // Obtain quota for target account let src_account_id = src_mailbox.id.account_id; @@ -400,7 +441,7 @@ impl SessionData { let mut train_batch = BatchBuilder::new(); let mut did_train = false; train_batch.with_account_id(src_account_id); - for (id, imap_id) in ids { + 'next_message: for (id, imap_id) in ids { match self .server .copy_message( @@ -448,22 +489,27 @@ impl SessionData { { copied_ids.push((imap_id.uid, uid)); } else { - let data_ = if let Some(data_) = self - .get_message_data(dest_account_id, existing_id) - .await - .imap_ctx(&arguments.tag, trc::location!())? - { - data_ - } else { - continue; - }; - let data = data_ - .to_unarchived::() - .imap_ctx(&arguments.tag, trc::location!())?; + let mut retries = 0; + + loop { + let data_ = if let Some(data_) = self + .get_message_data(dest_account_id, existing_id) + .await + .imap_ctx(&arguments.tag, trc::location!())? + { + data_ + } else { + continue 'next_message; + }; + let data = data_ + .to_unarchived::() + .imap_ctx(&arguments.tag, trc::location!())?; + + if let Some(uid) = data.inner.message_uid(dest_mailbox_id) { + copied_ids.push((imap_id.uid, uid)); + break; + } - if let Some(uid) = data.inner.message_uid(dest_mailbox_id) { - copied_ids.push((imap_id.uid, uid)); - } else { let mut new_data = data.inner.to_builder(); new_data.add_mailbox(UidMailbox::new_unassigned(dest_mailbox_id)); @@ -504,15 +550,27 @@ impl SessionData { ) .imap_ctx(&arguments.tag, trc::location!())?; - dest_change_id = self + match self .server .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(dest_account_id)) - .imap_ctx(&arguments.tag, trc::location!())? - .into(); - - copied_ids.push((imap_id.uid, assigned_uid)); + { + Ok(change_id) => { + dest_change_id = change_id.into(); + copied_ids.push((imap_id.uid, assigned_uid)); + break; + } + Err(err) => { + retry_after_conflict( + err, + &mut retries, + &arguments.tag, + trc::location!(), + ) + .await? + } + } } } } @@ -554,21 +612,33 @@ impl SessionData { // Untag or delete emails if !destroy_ids.is_empty() { - let mut batch = BatchBuilder::new(); - self.email_untag_or_delete( - src_account_id, - src_mailbox.id.mailbox_id, - &destroy_ids, - &mut batch, - ) - .await - .imap_ctx(&arguments.tag, trc::location!())?; + let mut retries = 0; - self.server - .commit_batch(batch) + loop { + let mut batch = BatchBuilder::new(); + self.email_untag_or_delete( + src_account_id, + src_mailbox.id.mailbox_id, + &destroy_ids, + &mut batch, + ) .await .imap_ctx(&arguments.tag, trc::location!())?; + match self.server.commit_batch(batch).await { + Ok(_) => break, + Err(err) => { + retry_after_conflict( + err, + &mut retries, + &arguments.tag, + trc::location!(), + ) + .await? + } + } + } + did_move = true; } @@ -731,3 +801,24 @@ impl SessionData { } } } + +async fn retry_after_conflict( + err: trc::Error, + retries: &mut u32, + tag: &str, + location: &'static str, +) -> trc::Result<()> { + if !err.is_assertion_failure() { + Err(err).imap_ctx(tag, location) + } else if *retries < MAX_MOVE_RETRIES { + *retries += 1; + let backoff = rand::rng().random_range(MOVE_RETRY_BACKOFF_MS); + tokio::time::sleep(Duration::from_millis(backoff)).await; + Ok(()) + } else { + Err(trc::ImapEvent::Error + .into_err() + .details("Some messages were modified by another process.") + .id(tag.to_string())) + } +} diff --git a/crates/jmap-proto/Cargo.toml b/crates/jmap-proto/Cargo.toml index 0382fab..9ad5ad7 100644 --- a/crates/jmap-proto/Cargo.toml +++ b/crates/jmap-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap_proto" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/jmap-proto/src/method/import.rs b/crates/jmap-proto/src/method/import.rs index 535a191..dbeb488 100644 --- a/crates/jmap-proto/src/method/import.rs +++ b/crates/jmap-proto/src/method/import.rs @@ -12,7 +12,6 @@ use crate::{ email::{EmailProperty, EmailValue}, }, request::{ - MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::{MaybeIdReference, MaybeResultReference, ResultReference}, }, @@ -33,7 +32,7 @@ pub struct ImportEmailRequest { #[derive(Debug, Clone, Default)] pub struct ImportEmail { - pub blob_id: MaybeInvalid, + pub blob_id: MaybeIdReference, pub mailbox_ids: MaybeResultReference>>, pub keywords: Vec, pub received_at: Option, diff --git a/crates/jmap-proto/src/references/resolve.rs b/crates/jmap-proto/src/references/resolve.rs index f83a580..27f4259 100644 --- a/crates/jmap-proto/src/references/resolve.rs +++ b/crates/jmap-proto/src/references/resolve.rs @@ -392,6 +392,10 @@ impl ResolveReference for ImportEmailRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve email mailbox references for email in self.emails.values_mut() { + if let MaybeIdReference::Reference(ir) = &email.blob_id { + email.blob_id = MaybeIdReference::Id(response.eval_blob_id_reference(ir)?); + } + match &mut email.mailbox_ids { MaybeResultReference::Reference(reference) => { email.mailbox_ids = MaybeResultReference::Value( diff --git a/crates/jmap-proto/src/request/reference.rs b/crates/jmap-proto/src/request/reference.rs index 4460b12..0b18e3b 100644 --- a/crates/jmap-proto/src/request/reference.rs +++ b/crates/jmap-proto/src/request/reference.rs @@ -105,6 +105,12 @@ impl Default for MaybeResultReference { } } +impl Default for MaybeIdReference { + fn default() -> Self { + MaybeIdReference::Invalid(String::new()) + } +} + impl MaybeResultReference { pub fn unwrap(self) -> T { match self { diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index dde3e3b..3198a40 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 030fe2b..d2b3d18 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -18,7 +18,7 @@ use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, object::email::EmailProperty, - request::MaybeInvalid, + request::reference::MaybeIdReference, types::state::State, }; use mail_parser::{HeaderName, MessageParser}; @@ -128,7 +128,7 @@ impl EmailImport for Server { } } - let MaybeInvalid::Value(blob_id) = email.blob_id else { + let MaybeIdReference::Id(blob_id) = email.blob_id else { response.not_created.append( id, SetError::invalid_properties() diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index b18182e..49eb88d 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -854,8 +854,11 @@ impl EmailSet for Server { new_data.set_mailboxes( ids.into_expanded_boolean_set() .filter_map(|id| { - UidMailbox::new_unassigned( - id.try_into_property()?.try_into_id()?.document_id(), + let mailbox_id = + id.try_into_property()?.try_into_id()?.document_id(); + UidMailbox::new( + mailbox_id, + data.inner.message_uid(mailbox_id).unwrap_or(0), ) .into() }) diff --git a/crates/jmap/src/registry/mapping/bootstrap.rs b/crates/jmap/src/registry/mapping/bootstrap.rs index 5e305ff..3960da9 100644 --- a/crates/jmap/src/registry/mapping/bootstrap.rs +++ b/crates/jmap/src/registry/mapping/bootstrap.rs @@ -641,6 +641,7 @@ fn map_dns_server(dns_server: &DnsServerBootstrap) -> Option DnsServer::Ns1(inner.clone()).into(), DnsServerBootstrap::OracleCloud(inner) => DnsServer::OracleCloud(inner.clone()).into(), DnsServerBootstrap::Plesk(inner) => DnsServer::Plesk(inner.clone()).into(), + DnsServerBootstrap::PowerDns(inner) => DnsServer::PowerDns(inner.clone()).into(), DnsServerBootstrap::Safedns(inner) => DnsServer::Safedns(inner.clone()).into(), DnsServerBootstrap::Scaleway(inner) => DnsServer::Scaleway(inner.clone()).into(), DnsServerBootstrap::TencentCloud(inner) => DnsServer::TencentCloud(inner.clone()).into(), diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index e6a6db4..e88f62f 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ keywords = ["imap", "jmap", "smtp", "email", "mail", "webdav", "server"] categories = ["email"] # Upstream offers AGPL-3.0-only OR LicenseRef-SEL; inbuxa takes the AGPL only. license = "AGPL-3.0-only" -version = "0.16.23" +version = "0.16.24" edition = "2024" [[bin]] diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 1eaec11..7be7f60 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "managesieve" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/migration/Cargo.toml b/crates/migration/Cargo.toml index 75c0928..ddb8652 100644 --- a/crates/migration/Cargo.toml +++ b/crates/migration/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "migration" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index 379e51c..3a23975 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nlp" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/pop3/Cargo.toml b/crates/pop3/Cargo.toml index d850c69..de7c540 100644 --- a/crates/pop3/Cargo.toml +++ b/crates/pop3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pop3" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/registry/Cargo.toml b/crates/registry/Cargo.toml index 4adb8f3..f918a37 100644 --- a/crates/registry/Cargo.toml +++ b/crates/registry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "registry" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/registry/src/schema/enums.rs b/crates/registry/src/schema/enums.rs index b9ec2f5..bf10006 100644 --- a/crates/registry/src/schema/enums.rs +++ b/crates/registry/src/schema/enums.rs @@ -620,6 +620,7 @@ pub enum DnsServerBootstrapType { Vultr = 68, WebSupport = 69, YandexCloud = 70, + PowerDns = 71, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -696,6 +697,7 @@ pub enum DnsServerType { Vultr = 67, WebSupport = 68, YandexCloud = 69, + PowerDns = 70, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] diff --git a/crates/registry/src/schema/enums_impl.rs b/crates/registry/src/schema/enums_impl.rs index 312b179..45dab07 100644 --- a/crates/registry/src/schema/enums_impl.rs +++ b/crates/registry/src/schema/enums_impl.rs @@ -3023,6 +3023,7 @@ impl EnumImpl for DnsServerBootstrapType { b"Vultr" => DnsServerBootstrapType::Vultr, b"WebSupport" => DnsServerBootstrapType::WebSupport, b"YandexCloud" => DnsServerBootstrapType::YandexCloud, + b"PowerDns" => DnsServerBootstrapType::PowerDns, } .copied() } @@ -3100,6 +3101,7 @@ impl EnumImpl for DnsServerBootstrapType { DnsServerBootstrapType::Vultr => "Vultr", DnsServerBootstrapType::WebSupport => "WebSupport", DnsServerBootstrapType::YandexCloud => "YandexCloud", + DnsServerBootstrapType::PowerDns => "PowerDns", } } @@ -3180,11 +3182,12 @@ impl EnumImpl for DnsServerBootstrapType { 68 => Some(DnsServerBootstrapType::Vultr), 69 => Some(DnsServerBootstrapType::WebSupport), 70 => Some(DnsServerBootstrapType::YandexCloud), + 71 => Some(DnsServerBootstrapType::PowerDns), _ => None, } } - const COUNT: usize = 71; + const COUNT: usize = 72; } impl serde::Serialize for DnsServerBootstrapType { @@ -3281,6 +3284,7 @@ impl EnumImpl for DnsServerType { b"Vultr" => DnsServerType::Vultr, b"WebSupport" => DnsServerType::WebSupport, b"YandexCloud" => DnsServerType::YandexCloud, + b"PowerDns" => DnsServerType::PowerDns, } .copied() } @@ -3357,6 +3361,7 @@ impl EnumImpl for DnsServerType { DnsServerType::Vultr => "Vultr", DnsServerType::WebSupport => "WebSupport", DnsServerType::YandexCloud => "YandexCloud", + DnsServerType::PowerDns => "PowerDns", } } @@ -3436,11 +3441,12 @@ impl EnumImpl for DnsServerType { 67 => Some(DnsServerType::Vultr), 68 => Some(DnsServerType::WebSupport), 69 => Some(DnsServerType::YandexCloud), + 70 => Some(DnsServerType::PowerDns), _ => None, } } - const COUNT: usize = 70; + const COUNT: usize = 71; } impl serde::Serialize for DnsServerType { diff --git a/crates/registry/src/schema/properties.rs b/crates/registry/src/schema/properties.rs index 1580020..e7469df 100644 --- a/crates/registry/src/schema/properties.rs +++ b/crates/registry/src/schema/properties.rs @@ -1042,6 +1042,7 @@ pub enum Property { SentinelUsername = 914, Separator = 97, ServerHostname = 121, + ServerId = 934, Servers = 308, ServiceAccountJson = 316, ServiceName = 913, diff --git a/crates/registry/src/schema/properties_impl.rs b/crates/registry/src/schema/properties_impl.rs index 052ec52..461418d 100644 --- a/crates/registry/src/schema/properties_impl.rs +++ b/crates/registry/src/schema/properties_impl.rs @@ -1195,6 +1195,7 @@ impl EnumImpl for Property { b"sentinelUsername" => Property::SentinelUsername, b"separator" => Property::Separator, b"serverHostname" => Property::ServerHostname, + b"serverId" => Property::ServerId, b"servers" => Property::Servers, b"serviceAccountJson" => Property::ServiceAccountJson, b"serviceName" => Property::ServiceName, @@ -2134,6 +2135,7 @@ impl EnumImpl for Property { Property::SentinelUsername => "sentinelUsername", Property::Separator => "separator", Property::ServerHostname => "serverHostname", + Property::ServerId => "serverId", Property::Servers => "servers", Property::ServiceAccountJson => "serviceAccountJson", Property::ServiceName => "serviceName", @@ -3077,6 +3079,7 @@ impl EnumImpl for Property { 914 => Some(Property::SentinelUsername), 97 => Some(Property::Separator), 121 => Some(Property::ServerHostname), + 934 => Some(Property::ServerId), 308 => Some(Property::Servers), 316 => Some(Property::ServiceAccountJson), 913 => Some(Property::ServiceName), @@ -3227,7 +3230,7 @@ impl EnumImpl for Property { } } - const COUNT: usize = 934; + const COUNT: usize = 935; } impl serde::Serialize for Property { @@ -4493,6 +4496,7 @@ impl ObjectInner { ObjectInner::DnsServer(DnsServer::Vultr(obj)) => obj.member_tenant_id, ObjectInner::DnsServer(DnsServer::WebSupport(obj)) => obj.member_tenant_id, ObjectInner::DnsServer(DnsServer::YandexCloud(obj)) => obj.member_tenant_id, + ObjectInner::DnsServer(DnsServer::PowerDns(obj)) => obj.member_tenant_id, ObjectInner::Domain(obj) => obj.member_tenant_id, ObjectInner::MailingList(obj) => obj.member_tenant_id, ObjectInner::OAuthClient(obj) => obj.member_tenant_id, @@ -4595,6 +4599,7 @@ impl ObjectInner { ObjectInner::DnsServer(DnsServer::Vultr(obj)) => obj.member_tenant_id = Some(id), ObjectInner::DnsServer(DnsServer::WebSupport(obj)) => obj.member_tenant_id = Some(id), ObjectInner::DnsServer(DnsServer::YandexCloud(obj)) => obj.member_tenant_id = Some(id), + ObjectInner::DnsServer(DnsServer::PowerDns(obj)) => obj.member_tenant_id = Some(id), ObjectInner::Domain(obj) => obj.member_tenant_id = Some(id), ObjectInner::MailingList(obj) => obj.member_tenant_id = Some(id), ObjectInner::OAuthClient(obj) => obj.member_tenant_id = Some(id), diff --git a/crates/registry/src/schema/structs.rs b/crates/registry/src/schema/structs.rs index 15aaf74..7066a57 100644 --- a/crates/registry/src/schema/structs.rs +++ b/crates/registry/src/schema/structs.rs @@ -1457,6 +1457,7 @@ pub enum DnsServer { Vultr(DnsServerCloud), WebSupport(DnsServerWebSupport), YandexCloud(DnsServerYandexCloud), + PowerDns(DnsServerPowerDns), } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -1672,6 +1673,7 @@ pub enum DnsServerBootstrap { Vultr(DnsServerCloud), WebSupport(DnsServerWebSupport), YandexCloud(DnsServerYandexCloud), + PowerDns(DnsServerPowerDns), } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -2435,6 +2437,31 @@ pub struct DnsServerPorkbun { pub propagation_delay: Option, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct DnsServerPowerDns { + #[serde(rename = "apiKey")] + pub api_key: SecretKey, + #[serde(rename = "endpoint")] + pub endpoint: Option, + #[serde(rename = "serverId")] + pub server_id: Option, + #[serde(rename = "description")] + pub description: String, + #[serde(rename = "memberTenantId")] + pub member_tenant_id: Option, + #[serde(rename = "timeout")] + pub timeout: Duration, + #[serde(rename = "ttl")] + pub ttl: Duration, + #[serde(rename = "pollingInterval")] + pub polling_interval: Duration, + #[serde(rename = "propagationTimeout")] + pub propagation_timeout: Duration, + #[serde(rename = "propagationDelay")] + pub propagation_delay: Option, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerRoute53 { diff --git a/crates/registry/src/schema/structs_impl.rs b/crates/registry/src/schema/structs_impl.rs index 1ef2839..529a9ec 100644 --- a/crates/registry/src/schema/structs_impl.rs +++ b/crates/registry/src/schema/structs_impl.rs @@ -10559,6 +10559,7 @@ impl ObjectImpl for DnsServer { DnsServer::Vultr(inner) => inner.validate(errors), DnsServer::WebSupport(inner) => inner.validate(errors), DnsServer::YandexCloud(inner) => inner.validate(errors), + DnsServer::PowerDns(inner) => inner.validate(errors), } } @@ -10772,6 +10773,9 @@ impl ObjectImpl for DnsServer { DnsServer::YandexCloud(object) => { object.index(i); } + DnsServer::PowerDns(object) => { + object.index(i); + } } } } @@ -11064,6 +11068,10 @@ impl Pickle for DnsServer { 69u16.pickle(out); inner.pickle(out); } + DnsServer::PowerDns(inner) => { + 70u16.pickle(out); + inner.pickle(out); + } } } @@ -11139,6 +11147,7 @@ impl Pickle for DnsServer { 67 => Pickle::unpickle(stream).map(DnsServer::Vultr), 68 => Pickle::unpickle(stream).map(DnsServer::WebSupport), 69 => Pickle::unpickle(stream).map(DnsServer::YandexCloud), + 70 => Pickle::unpickle(stream).map(DnsServer::PowerDns), _ => None, } } @@ -11635,6 +11644,13 @@ impl IntoValue for DnsServer { .insert_unchecked(Property::Type, JmapValue::Str("YandexCloud".into())); obj } + DnsServer::PowerDns(obj) => { + let mut obj = obj.into_value(); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("PowerDns".into())); + obj + } } } } @@ -11719,6 +11735,7 @@ impl RegistryJsonPatch for DnsServer { DnsServerType::Vultr => *self = DnsServer::Vultr(Default::default()), DnsServerType::WebSupport => *self = DnsServer::WebSupport(Default::default()), DnsServerType::YandexCloud => *self = DnsServer::YandexCloud(Default::default()), + DnsServerType::PowerDns => *self = DnsServer::PowerDns(Default::default()), } } match self { @@ -11792,6 +11809,7 @@ impl RegistryJsonPatch for DnsServer { DnsServer::Vultr(inner) => inner.patch(pointer, value), DnsServer::WebSupport(inner) => inner.patch(pointer, value), DnsServer::YandexCloud(inner) => inner.patch(pointer, value), + DnsServer::PowerDns(inner) => inner.patch(pointer, value), } } } @@ -11869,6 +11887,7 @@ impl DnsServer { DnsServer::Vultr(_) => DnsServerType::Vultr, DnsServer::WebSupport(_) => DnsServerType::WebSupport, DnsServer::YandexCloud(_) => DnsServerType::YandexCloud, + DnsServer::PowerDns(_) => DnsServerType::PowerDns, } } } @@ -12704,6 +12723,7 @@ impl DnsServerBootstrap { DnsServerBootstrap::Vultr(inner) => inner.validate(errors), DnsServerBootstrap::WebSupport(inner) => inner.validate(errors), DnsServerBootstrap::YandexCloud(inner) => inner.validate(errors), + DnsServerBootstrap::PowerDns(inner) => inner.validate(errors), } } } @@ -12999,6 +13019,10 @@ impl Pickle for DnsServerBootstrap { 70u16.pickle(out); inner.pickle(out); } + DnsServerBootstrap::PowerDns(inner) => { + 71u16.pickle(out); + inner.pickle(out); + } } } @@ -13075,6 +13099,7 @@ impl Pickle for DnsServerBootstrap { 68 => Pickle::unpickle(stream).map(DnsServerBootstrap::Vultr), 69 => Pickle::unpickle(stream).map(DnsServerBootstrap::WebSupport), 70 => Pickle::unpickle(stream).map(DnsServerBootstrap::YandexCloud), + 71 => Pickle::unpickle(stream).map(DnsServerBootstrap::PowerDns), _ => None, } } @@ -13576,6 +13601,13 @@ impl IntoValue for DnsServerBootstrap { .insert_unchecked(Property::Type, JmapValue::Str("YandexCloud".into())); obj } + DnsServerBootstrap::PowerDns(obj) => { + let mut obj = obj.into_value(); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("PowerDns".into())); + obj + } } } } @@ -13793,6 +13825,9 @@ impl RegistryJsonPatch for DnsServerBootstrap { DnsServerBootstrapType::YandexCloud => { *self = DnsServerBootstrap::YandexCloud(Default::default()) } + DnsServerBootstrapType::PowerDns => { + *self = DnsServerBootstrap::PowerDns(Default::default()) + } } } match self { @@ -13867,6 +13902,7 @@ impl RegistryJsonPatch for DnsServerBootstrap { DnsServerBootstrap::Vultr(inner) => inner.patch(pointer, value), DnsServerBootstrap::WebSupport(inner) => inner.patch(pointer, value), DnsServerBootstrap::YandexCloud(inner) => inner.patch(pointer, value), + DnsServerBootstrap::PowerDns(inner) => inner.patch(pointer, value), } } } @@ -13945,6 +13981,7 @@ impl DnsServerBootstrap { DnsServerBootstrap::Vultr(_) => DnsServerBootstrapType::Vultr, DnsServerBootstrap::WebSupport(_) => DnsServerBootstrapType::WebSupport, DnsServerBootstrap::YandexCloud(_) => DnsServerBootstrapType::YandexCloud, + DnsServerBootstrap::PowerDns(_) => DnsServerBootstrapType::PowerDns, } } } @@ -18228,6 +18265,148 @@ impl RegistryJsonPropertyPatch for DnsServerPorkbun { } } +impl DnsServerPowerDns { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.api_key; + value.validate(errors); + if let Some(value) = &self.endpoint { + if value.is_empty() { + errors.push(ValidationError::required(Property::Endpoint)); + } + } + if let Some(value) = &self.server_id { + if value.is_empty() { + errors.push(ValidationError::required(Property::ServerId)); + } + } + let value = &self.description; + if value.is_empty() { + errors.push(ValidationError::required(Property::Description)); + } + if let Some(value) = &self.member_tenant_id { + if !value.is_valid() { + errors.push(ValidationError::required(Property::MemberTenantId)); + } + } + errors.len() == neb + } + + fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { + i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); + if let Some(value) = &self.member_tenant_id { + i.search(Property::MemberTenantId, value); + } + } +} + +impl Pickle for DnsServerPowerDns { + fn pickle(&self, out: &mut Vec) { + self.api_key.pickle(out); + self.endpoint.pickle(out); + self.server_id.pickle(out); + self.description.pickle(out); + self.member_tenant_id.pickle(out); + self.timeout.pickle(out); + self.ttl.pickle(out); + self.polling_interval.pickle(out); + self.propagation_timeout.pickle(out); + self.propagation_delay.pickle(out); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let mut this = Self::default(); + this.api_key = Pickle::unpickle(stream)?; + this.endpoint = Pickle::unpickle(stream)?; + this.server_id = Pickle::unpickle(stream)?; + this.description = Pickle::unpickle(stream)?; + this.member_tenant_id = Pickle::unpickle(stream)?; + this.timeout = Pickle::unpickle(stream)?; + this.ttl = Pickle::unpickle(stream)?; + this.polling_interval = Pickle::unpickle(stream)?; + this.propagation_timeout = Pickle::unpickle(stream)?; + this.propagation_delay = Pickle::unpickle(stream)?; + Some(this) + } +} + +impl Default for DnsServerPowerDns { + fn default() -> Self { + Self { + api_key: Default::default(), + endpoint: Default::default(), + server_id: Default::default(), + description: Default::default(), + member_tenant_id: Default::default(), + timeout: Duration::from_millis(30000), + ttl: Duration::from_millis(300000), + polling_interval: Duration::from_millis(15000), + propagation_timeout: Duration::from_millis(60000), + propagation_delay: Default::default(), + } + } +} + +impl IntoValue for DnsServerPowerDns { + fn into_value(self) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(12); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); + map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); + map.insert_unchecked(Property::ServerId, self.server_id.into_value()); + map.insert_unchecked(Property::Description, self.description.into_value()); + map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); + map.insert_unchecked(Property::Timeout, self.timeout.into_value()); + map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked( + Property::PollingInterval, + self.polling_interval.into_value(), + ); + map.insert_unchecked( + Property::PropagationTimeout, + self.propagation_timeout.into_value(), + ); + map.insert_unchecked( + Property::PropagationDelay, + self.propagation_delay.into_value(), + ); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for DnsServerPowerDns { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::ApiKey) => self.api_key.patch(pointer, value), + Some(Property::Endpoint) => self + .endpoint + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::ServerId) => self + .server_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Description) => self + .description + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::MemberTenantId) => self + .member_tenant_id + .patch(pointer.assert_can_set_tenant()?, value), + Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::Ttl) => self.ttl.patch(pointer, value), + Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), + Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + impl DnsServerRoute53 { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); diff --git a/crates/registry/src/utils/mod.rs b/crates/registry/src/utils/mod.rs index 94c21c6..ee62664 100644 --- a/crates/registry/src/utils/mod.rs +++ b/crates/registry/src/utils/mod.rs @@ -14,6 +14,7 @@ pub mod dkim; pub mod http; pub mod report; pub mod secret; +pub mod spam; pub mod task; impl Roles { diff --git a/crates/registry/src/utils/spam.rs b/crates/registry/src/utils/spam.rs new file mode 100644 index 0000000..bd302e4 --- /dev/null +++ b/crates/registry/src/utils/spam.rs @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::schema::prelude::{SpamDnsblServer, SpamRule}; + +impl SpamRule { + pub fn enable(&self) -> bool { + match self { + SpamRule::Any(rule) => rule.enable, + SpamRule::Url(rule) => rule.enable, + SpamRule::Domain(rule) => rule.enable, + SpamRule::Email(rule) => rule.enable, + SpamRule::Ip(rule) => rule.enable, + SpamRule::Header(rule) => rule.enable, + SpamRule::Body(rule) => rule.enable, + } + } + + pub fn set_enable(&mut self, enable: bool) { + match self { + SpamRule::Any(rule) => rule.enable = enable, + SpamRule::Url(rule) => rule.enable = enable, + SpamRule::Domain(rule) => rule.enable = enable, + SpamRule::Email(rule) => rule.enable = enable, + SpamRule::Ip(rule) => rule.enable = enable, + SpamRule::Header(rule) => rule.enable = enable, + SpamRule::Body(rule) => rule.enable = enable, + } + } +} + +impl SpamDnsblServer { + pub fn enable(&self) -> bool { + match self { + SpamDnsblServer::Any(server) => server.enable, + SpamDnsblServer::Url(server) => server.enable, + SpamDnsblServer::Domain(server) => server.enable, + SpamDnsblServer::Email(server) => server.enable, + SpamDnsblServer::Ip(server) => server.enable, + SpamDnsblServer::Header(server) => server.enable, + SpamDnsblServer::Body(server) => server.enable, + } + } + + pub fn set_enable(&mut self, enable: bool) { + match self { + SpamDnsblServer::Any(server) => server.enable = enable, + SpamDnsblServer::Url(server) => server.enable = enable, + SpamDnsblServer::Domain(server) => server.enable = enable, + SpamDnsblServer::Email(server) => server.enable = enable, + SpamDnsblServer::Ip(server) => server.enable = enable, + SpamDnsblServer::Header(server) => server.enable = enable, + SpamDnsblServer::Body(server) => server.enable = enable, + } + } +} diff --git a/crates/scim-proto/Cargo.toml b/crates/scim-proto/Cargo.toml index 099ee79..b66bf6e 100644 --- a/crates/scim-proto/Cargo.toml +++ b/crates/scim-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scim-proto" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/scim/Cargo.toml b/crates/scim/Cargo.toml index 0636477..9fa6469 100644 --- a/crates/scim/Cargo.toml +++ b/crates/scim/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scim" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 8e98b97..d019a55 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "services" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/services/src/state_manager/http.rs b/crates/services/src/state_manager/http.rs index 96f01eb..39987ba 100644 --- a/crates/services/src/state_manager/http.rs +++ b/crates/services/src/state_manager/http.rs @@ -9,7 +9,7 @@ use super::{ ece::{ECE_WEBPUSH_MAX_PLAINTEXT_SIZE, WEBPUSH_MAX_BODY_SIZE, ece_encrypt}, email_push::build_email_push_object, }; -use crate::state_manager::PushRegistration; +use crate::state_manager::{PushBatch, PushRegistration}; use calcard::jscalendar::JSCalendarDateTime; use common::{Server, ipc::PushNotification, network::webpush::Vapid}; use email::push::{PushSubscription, Urgency}; @@ -29,7 +29,7 @@ use std::time::{Duration, Instant}; use store::write::now; use tokio::sync::mpsc; use trc::PushSubscriptionEvent; -use types::{id::Id, type_state::DataType}; +use types::id::Id; use utils::map::vec_map::VecMap; const MAX_ERROR_RESPONSE_LEN: usize = 1024; @@ -48,34 +48,29 @@ impl PushRegistration { pub fn send( &mut self, id: Id, + push_client: &Client, push_tx: mpsc::Sender, push_timeout: Duration, server: Server, ) { let subscription = self.server.clone(); - let push_client = self.client.clone(); - let notifications = std::mem::take(&mut self.notifications); + let push_client = push_client.clone(); + let batch = std::mem::take(&mut self.pending); self.in_flight = true; self.last_request = Instant::now(); tokio::spawn(async move { - let mut changed: VecMap> = VecMap::new(); + let vapid = server.core.jmap.vapid.as_ref(); let mut email_pushes: VecMap = VecMap::new(); - let mut failed_state_change = false; + let mut failed = PushBatch::default(); let mut failed_email_pushes = Vec::new(); let mut failed_calendar_alerts = Vec::new(); - for notification in ¬ifications { + for notification in &batch.notifications { match notification { - PushNotification::StateChange(state_change) => { - for type_state in state_change.types { - changed - .get_mut_or_insert(state_change.account_id.into()) - .set(type_state, State::Exact(state_change.change_id)); - } - } + PushNotification::StateChange(_) => {} PushNotification::CalendarAlert(calendar_alert) => { let payload = PushObject::CalendarAlert { account_id: calendar_alert.account_id.into(), @@ -86,12 +81,12 @@ impl PushRegistration { }), alert_id: calendar_alert.alert_id.clone(), }; - if !http_request( + if !post_object( &push_client, &subscription, - serde_json::to_string(&payload).unwrap().into_bytes(), + &payload, push_timeout, - server.core.jmap.vapid.as_ref(), + vapid, Urgency::Normal, ) .await @@ -155,18 +150,23 @@ impl PushRegistration { } } - if !changed.is_empty() { - failed_state_change = !http_request( + if !batch.state_changes.is_empty() { + let payload = PushObject::StateChange { + changed: batch.state_changes, + }; + if !post_object( &push_client, &subscription, - serde_json::to_string(&PushObject::StateChange { changed }) - .unwrap() - .into_bytes(), + &payload, push_timeout, - server.core.jmap.vapid.as_ref(), + vapid, Urgency::Normal, ) - .await; + .await + && let PushObject::StateChange { changed } = payload + { + failed.state_changes = changed; + } } for (account_id, email_push) in email_pushes { @@ -180,12 +180,12 @@ impl PushRegistration { state: email_push.change_id.map(State::Exact), }; - if !http_request( + if !post_object( &push_client, &subscription, - serde_json::to_string(&payload).unwrap().into_bytes(), + &payload, push_timeout, - server.core.jmap.vapid.as_ref(), + vapid, email_push.urgency, ) .await @@ -194,44 +194,26 @@ impl PushRegistration { } } - let result = if !failed_state_change + let result = if failed.state_changes.is_empty() && failed_email_pushes.is_empty() && failed_calendar_alerts.is_empty() { Event::DeliverySuccess { id } } else { - let mut failed_notifications = Vec::with_capacity( - failed_state_change as usize - + failed_email_pushes.len() - + failed_calendar_alerts.len(), - ); - - for notification in notifications { - match ¬ification { - PushNotification::StateChange(_) => { - if failed_state_change { - failed_notifications.push(notification); - } - } + failed.notifications = batch + .notifications + .into_iter() + .filter(|notification| match notification { + PushNotification::StateChange(_) => false, PushNotification::EmailPush(email_push) => { - if failed_email_pushes.contains(&email_push.account_id) { - failed_notifications.push(notification); - } + failed_email_pushes.contains(&email_push.account_id) } - PushNotification::CalendarAlert(calendar_alert) => { - if failed_calendar_alerts - .contains(&(calendar_alert.account_id, calendar_alert.event_id)) - { - failed_notifications.push(notification); - } - } - } - } + PushNotification::CalendarAlert(calendar_alert) => failed_calendar_alerts + .contains(&(calendar_alert.account_id, calendar_alert.event_id)), + }) + .collect(); - Event::DeliveryFailure { - id, - notifications: failed_notifications, - } + Event::DeliveryFailure { id, failed } }; push_tx.send(result).await.ok(); @@ -239,6 +221,38 @@ impl PushRegistration { } } +async fn post_object( + push_client: &Client, + subscription: &PushSubscription, + object: &PushObject, + push_timeout: Duration, + vapid: Option<&Vapid>, + urgency: Urgency, +) -> bool { + match serde_json::to_vec(object) { + Ok(body) => { + http_request( + push_client, + subscription, + body, + push_timeout, + vapid, + urgency, + ) + .await + } + Err(err) => { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Failed to serialize push object", + Url = subscription.url.to_string(), + Reason = err.to_string() + ); + true + } + } +} + pub(crate) fn build_push_client() -> Client { utils::http::http_client_builder(cfg!(feature = "test_mode")) .redirect(Policy::custom(|attempt| match attempt.previous().last() { diff --git a/crates/services/src/state_manager/manager.rs b/crates/services/src/state_manager/manager.rs index d744982..b4e5f5b 100644 --- a/crates/services/src/state_manager/manager.rs +++ b/crates/services/src/state_manager/manager.rs @@ -14,7 +14,7 @@ use common::{ }; use std::{sync::Arc, time::Instant}; use store::ahash::AHashMap; -use tokio::sync::mpsc; +use tokio::sync::mpsc::{self, error::TrySendError}; use trc::ServerEvent; #[derive(Default)] @@ -99,7 +99,7 @@ pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver { // Publish event to cluster if broadcast - && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone() + && let Some(broadcast_tx) = inner.ipc.broadcast_tx.as_ref() && broadcast_tx .send(BroadcastEvent::PushNotification(notification.clone())) .await @@ -117,26 +117,30 @@ pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver {} + Err(TrySendError::Full(notification)) => { + let subscriber_tx = subscriber.tx.clone(); - tokio::spawn(async move { - // Timeout after 500ms in case there is a blocked client - if subscriber_tx - .send_timeout(notification, SEND_TIMEOUT) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Details = - "Error sending state change to subscriber.", - CausedBy = trc::location!() - ); - } - }); - } else { - purge_needed = true; + tokio::spawn(async move { + // Timeout after 500ms in case there is a blocked client + if subscriber_tx + .send_timeout(notification, SEND_TIMEOUT) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = + "Error sending state change to subscriber.", + CausedBy = trc::location!() + ); + } + }); + } + Err(TrySendError::Closed(_)) => { + purge_needed = true; + } } } } @@ -159,7 +163,7 @@ pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver { // Publish event to cluster if broadcast - && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone() + && let Some(broadcast_tx) = inner.ipc.broadcast_tx.as_ref() && broadcast_tx .send(BroadcastEvent::PushServerUpdate(account_id)) .await diff --git a/crates/services/src/state_manager/mod.rs b/crates/services/src/state_manager/mod.rs index 6b927b0..087db23 100644 --- a/crates/services/src/state_manager/mod.rs +++ b/crates/services/src/state_manager/mod.rs @@ -14,14 +14,14 @@ pub mod push; use common::ipc::PushNotification; use email::push::PushSubscription; -use reqwest::Client; +use jmap_proto::types::state::State; use std::{ sync::Arc, time::{Duration, Instant}, }; use tokio::sync::mpsc; use types::{id::Id, type_state::DataType}; -use utils::map::bitmap::Bitmap; +use utils::map::{bitmap::Bitmap, vec_map::VecMap}; const PURGE_EVERY: Duration = Duration::from_secs(3600); const SEND_TIMEOUT: Duration = Duration::from_millis(500); @@ -40,26 +40,22 @@ pub struct PushRegistration { member_account_ids: Vec, num_attempts: u32, last_request: Instant, - notifications: Vec, + pending: PushBatch, in_flight: bool, - client: Client, +} + +#[derive(Debug, Default)] +pub struct PushBatch { + state_changes: VecMap>, + notifications: Vec, } #[derive(Debug)] pub enum Event { - Push { - notification: PushNotification, - }, - Update { - account_id: u32, - }, - DeliverySuccess { - id: Id, - }, - DeliveryFailure { - id: Id, - notifications: Vec, - }, + Push { notification: PushNotification }, + Update { account_id: u32 }, + DeliverySuccess { id: Id }, + DeliveryFailure { id: Id, failed: PushBatch }, Reset, } @@ -68,3 +64,130 @@ impl IpcSubscriber { !self.tx.is_closed() } } + +impl PushBatch { + pub fn push(&mut self, notification: PushNotification) { + match notification { + PushNotification::StateChange(state_change) => { + if !state_change.types.is_empty() { + let states = self + .state_changes + .get_mut_or_insert(Id::from(state_change.account_id)); + for data_type in state_change.types { + merge_state(states, data_type, state_change.change_id); + } + } + } + notification => self.notifications.push(notification), + } + } + + pub fn merge_failed(&mut self, failed: PushBatch) { + for (account_id, failed_states) in failed.state_changes { + let states = self.state_changes.get_mut_or_insert(account_id); + for (data_type, state) in failed_states { + if let State::Exact(change_id) = state { + merge_state(states, data_type, change_id); + } + } + } + + if !failed.notifications.is_empty() { + let mut notifications = failed.notifications; + notifications.append(&mut self.notifications); + self.notifications = notifications; + } + } + + pub fn is_empty(&self) -> bool { + self.state_changes.is_empty() && self.notifications.is_empty() + } + + pub fn clear(&mut self) { + self.state_changes.clear(); + self.notifications.clear(); + } +} + +fn merge_state(states: &mut VecMap, data_type: DataType, change_id: u64) { + match states.get_mut(&data_type) { + Some(State::Exact(current)) if *current >= change_id => {} + Some(state) => *state = State::Exact(change_id), + None => states.append(data_type, State::Exact(change_id)), + } +} + +#[cfg(test)] +mod tests { + use super::PushBatch; + use common::ipc::{EmailPush, PushNotification}; + use jmap_proto::types::state::State; + use types::{ + id::Id, + type_state::{DataType, StateChange}, + }; + use utils::map::bitmap::Bitmap; + + fn state_change(change_id: u64, types: [DataType; N]) -> PushNotification { + PushNotification::StateChange(StateChange { + account_id: 1, + change_id, + types: Bitmap::from_iter(types), + }) + } + + fn email_push(email_id: u32) -> PushNotification { + PushNotification::EmailPush(EmailPush { + account_id: 1, + email_id, + change_id: email_id.into(), + }) + } + + #[test] + fn batch_keeps_newest_state_per_type() { + let mut batch = PushBatch::default(); + batch.push(state_change(5, [DataType::Email])); + batch.push(state_change(7, [DataType::Email, DataType::Mailbox])); + batch.push(state_change(6, [DataType::Mailbox])); + + let mut failed = PushBatch::default(); + failed.push(state_change( + 4, + [DataType::Email, DataType::Mailbox, DataType::Thread], + )); + batch.merge_failed(failed); + + let states = batch.state_changes.get(&Id::from(1u32)).unwrap(); + assert_eq!(states.len(), 3); + assert_eq!(states.get(&DataType::Email), Some(&State::Exact(7))); + assert_eq!(states.get(&DataType::Mailbox), Some(&State::Exact(7))); + assert_eq!(states.get(&DataType::Thread), Some(&State::Exact(4))); + + assert!(!batch.is_empty()); + batch.clear(); + assert!(batch.is_empty()); + } + + #[test] + fn failed_notifications_are_retried_first() { + let mut batch = PushBatch::default(); + batch.push(email_push(3)); + + let mut failed = PushBatch::default(); + failed.push(email_push(1)); + failed.push(email_push(2)); + batch.merge_failed(failed); + + let email_ids = batch + .notifications + .iter() + .filter_map(|notification| match notification { + PushNotification::EmailPush(email_push) => Some(email_push.email_id), + _ => None, + }) + .collect::>(); + assert_eq!(email_ids, [1, 2, 3]); + assert!(batch.state_changes.is_empty()); + } +} diff --git a/crates/services/src/state_manager/push.rs b/crates/services/src/state_manager/push.rs index a2a9026..ecc585a 100644 --- a/crates/services/src/state_manager/push.rs +++ b/crates/services/src/state_manager/push.rs @@ -8,13 +8,14 @@ use super::{ Event, http::{build_push_client, http_request}, }; -use crate::state_manager::PushRegistration; +use crate::state_manager::{PushBatch, PushRegistration}; use common::{ BuildServer, IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, Server, auth::BuildAccessToken, ipc::{PushEvent, PushNotification}, }; use email::push::{PushSubscription, PushSubscriptions, Urgency}; +use reqwest::Client; use std::{ collections::hash_map::Entry, sync::Arc, @@ -36,7 +37,10 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { tokio::spawn(async move { let mut push_servers: AHashMap = AHashMap::default(); let mut account_push_ids: AHashMap> = AHashMap::default(); - let mut last_verify: AHashMap = AHashMap::default(); + let mut last_verify: AHashMap = AHashMap::default(); + let mut pending_verify: AHashMap)> = + AHashMap::default(); + let mut next_verify: Option = None; let mut last_retry = Instant::now(); let mut retry_timeout = LONG_1Y_SLUMBER; let mut retry_ids = AHashSet::default(); @@ -90,10 +94,9 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { last_request: Instant::now() - (server.core.jmap.push_throttle + Duration::from_millis(1)), - notifications: Vec::new(), + pending: PushBatch::default(), server: subscription.clone(), in_flight: false, - client: push_client.clone(), }, ); } @@ -129,8 +132,39 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { } loop { + if let Some(verify_due) = next_verify { + let current_instant = Instant::now(); + if verify_due <= current_instant { + let server = inner.build_server(); + let push_timeout = server.core.jmap.push_timeout; + let current_time = now(); + next_verify = None; + pending_verify.retain(|account_id, (verify_due, subscription)| { + if *verify_due > current_instant { + next_verify = + Some(next_verify.map_or(*verify_due, |next| next.min(*verify_due))); + true + } else { + if subscription.expires > current_time { + last_verify.insert(*account_id, (current_instant, subscription.id)); + send_verification( + &push_client, + subscription.clone(), + &server, + push_timeout, + ); + } + false + } + }); + } + } + // Wait for the next event or timeout - let event_or_timeout = tokio::time::timeout(retry_timeout, push_rx.recv()).await; + let wait_timeout = next_verify.map_or(retry_timeout, |verify_due| { + retry_timeout.min(verify_due.saturating_duration_since(Instant::now())) + }); + let event_or_timeout = tokio::time::timeout(wait_timeout, push_rx.recv()).await; // Load settings let server = inner.build_server(); @@ -193,10 +227,9 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { num_attempts: 0, last_request: Instant::now() - (push_throttle + Duration::from_millis(1)), - notifications: Vec::new(), + pending: PushBatch::default(), server: subscription.clone(), in_flight: false, - client: push_client.clone(), }); } } @@ -213,52 +246,56 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { #[cfg(feature = "test_mode")] if subscription.url.contains("skip_checks") { - last_verify.insert( - account_id, - current_time - (push_verify_timeout + Duration::from_millis(1)), - ); + last_verify.remove(&account_id); } - if last_verify + match last_verify .get(&account_id) - .map(|last_verify| { - current_time - *last_verify > push_verify_timeout + .map(|(verified_at, verified_id)| { + (*verified_at + push_verify_timeout, *verified_id) }) - .unwrap_or(true) + .filter(|(verify_due, _)| *verify_due >= current_time) { - let core = server.core.clone(); - let push_client = push_client.clone(); - tokio::spawn(async move { - http_request( + None => { + last_verify.retain(|_, (verified_at, _)| { + current_time.duration_since(*verified_at) + <= push_verify_timeout + }); + last_verify.insert(account_id, (current_time, subscription.id)); + pending_verify.remove(&account_id); + send_verification( &push_client, - &subscription, - format!( - concat!( - "{{\"@type\":\"PushVerification\",", - "\"pushSubscriptionId\":\"{}\",", - "\"verificationCode\":\"{}\"}}" - ), - Id::from(subscription.id), - subscription.verification_code - ) - .into_bytes(), + subscription, + &server, push_timeout, - core.jmap.vapid.as_ref(), - Urgency::Normal, - ) - .await; - }); - - last_verify.insert(account_id, current_time); - } else { - trc::event!( - PushSubscription(PushSubscriptionEvent::Error), - Details = "Failed to verify push subscription", - Url = subscription.url.clone(), - AccountId = account_id, - Reason = "Too many requests" - ); + ); + } + Some((_, verified_id)) if verified_id == subscription.id => { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Failed to verify push subscription", + Url = subscription.url.clone(), + AccountId = account_id, + Reason = "Too many requests" + ); + pending_verify.remove(&account_id); + } + Some((verify_due, _)) => { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Push subscription verification deferred", + Url = subscription.url.clone(), + AccountId = account_id, + Reason = "Too many requests" + ); + next_verify = Some( + next_verify.map_or(verify_due, |next| next.min(verify_due)), + ); + pending_verify.insert(account_id, (verify_due, subscription)); + } } + } else { + pending_verify.remove(&account_id); } // Update subscriptions @@ -342,7 +379,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { ); } - subscription.notifications.push(notification); + subscription.pending.push(notification); let last_request = subscription.last_request.elapsed(); if !subscription.in_flight @@ -354,6 +391,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { { subscription.send( *id, + &push_client, push_tx.clone(), push_timeout, server.clone(), @@ -402,19 +440,25 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { Event::Reset => { push_servers.clear(); account_push_ids.clear(); + pending_verify.clear(); + next_verify = None; } Event::DeliverySuccess { id } => { if let Some(subscription) = push_servers.get_mut(&id) { subscription.num_attempts = 0; subscription.in_flight = false; - retry_ids.remove(&id); + if subscription.pending.is_empty() { + retry_ids.remove(&id); + } else { + retry_ids.insert(id); + } } } - Event::DeliveryFailure { id, notifications } => { + Event::DeliveryFailure { id, failed } => { if let Some(subscription) = push_servers.get_mut(&id) { subscription.last_request = Instant::now(); subscription.num_attempts += 1; - subscription.notifications.extend(notifications); + subscription.pending.merge_failed(failed); subscription.in_flight = false; retry_ids.insert(id); } @@ -430,52 +474,46 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let last_retry_elapsed = last_retry.elapsed(); if last_retry_elapsed >= push_retry_interval { - let mut remove_ids = Vec::with_capacity(retry_ids.len()); + retry_ids.retain(|retry_id| { + let Some(subscription) = push_servers.get_mut(retry_id) else { + return false; + }; + let last_request = subscription.last_request.elapsed(); + let is_due = !subscription.in_flight + && ((subscription.num_attempts == 0 && last_request >= push_throttle) + || (subscription.num_attempts > 0 + && last_request >= push_attempt_interval)); + if !is_due { + return true; + } - for retry_id in &retry_ids { - if let Some(subscription) = push_servers.get_mut(retry_id) { - let last_request = subscription.last_request.elapsed(); - - if !subscription.in_flight - && ((subscription.num_attempts == 0 - && last_request >= push_throttle) - || (subscription.num_attempts > 0 - && last_request >= push_attempt_interval)) - { - if subscription.num_attempts < push_attempts_max { - subscription.send( - *retry_id, - push_tx.clone(), - push_timeout, - server.clone(), - ); - } else { - trc::event!( - PushSubscription(PushSubscriptionEvent::Error), - Details = "Failed to deliver push subscription", - Url = subscription.server.url.clone(), - Reason = "Too many failed attempts" - ); - - subscription.notifications.clear(); - subscription.num_attempts = 0; - } - remove_ids.push(*retry_id); - } + if subscription.num_attempts < push_attempts_max { + subscription.send( + *retry_id, + &push_client, + push_tx.clone(), + push_timeout, + server.clone(), + ); } else { - remove_ids.push(*retry_id); - } - } + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Failed to deliver push subscription", + Url = subscription.server.url.clone(), + Reason = "Too many failed attempts" + ); - if remove_ids.len() < retry_ids.len() { - for remove_id in remove_ids { - retry_ids.remove(&remove_id); + subscription.pending.clear(); + subscription.num_attempts = 0; } + false + }); + + if retry_ids.is_empty() { + LONG_1Y_SLUMBER + } else { last_retry = Instant::now(); push_retry_interval - } else { - retry_ids.clear(); - LONG_1Y_SLUMBER } } else { push_retry_interval - last_retry_elapsed @@ -489,6 +527,36 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { push_tx_ } +fn send_verification( + push_client: &Client, + subscription: Arc, + server: &Server, + push_timeout: Duration, +) { + let core = server.core.clone(); + let push_client = push_client.clone(); + tokio::spawn(async move { + http_request( + &push_client, + &subscription, + format!( + concat!( + "{{\"@type\":\"PushVerification\",", + "\"pushSubscriptionId\":\"{}\",", + "\"verificationCode\":\"{}\"}}" + ), + Id::from(subscription.id), + subscription.verification_code + ) + .into_bytes(), + push_timeout, + core.jmap.vapid.as_ref(), + Urgency::Normal, + ) + .await; + }); +} + async fn load_push_subscriptions( server: &Server, account_id: u32, diff --git a/crates/services/src/task_manager/spam_classifier.rs b/crates/services/src/task_manager/spam_classifier.rs index dd5f41e..f4068f5 100644 --- a/crates/services/src/task_manager/spam_classifier.rs +++ b/crates/services/src/task_manager/spam_classifier.rs @@ -15,13 +15,13 @@ use common::{ use registry::{ schema::{ enums::TaskSpamFilterMaintenanceType, - prelude::ObjectType, + prelude::{Object, ObjectInner, ObjectType}, structs::{ HttpLookup, MemoryLookupKey, SpamDnsblServer, SpamFileExtension, SpamRule, SpamTag, TaskSpamFilterMaintenance, }, }, - types::EnumImpl, + types::{EnumImpl, ObjectImpl}, }; use spam_filter::modules::classifier::SpamClassifier; use std::time::{Duration, Instant}; @@ -100,13 +100,90 @@ struct Rules { file_exts: Vec, } -#[derive(Default)] +trait UpstreamObject: ObjectImpl + PartialEq + From + Into { + fn replacement_for(self, _local: &Self) -> Option { + Some(self) + } + + // inbuxa: the object as its fingerprint sees it. Switching a rule on or + // off isn't an edit, so `enable` is left out. + fn without_enable(self) -> Self { + self + } +} + +impl UpstreamObject for SpamRule { + fn replacement_for(mut self, local: &Self) -> Option { + self.set_enable(local.enable()); + Some(self) + } + + fn without_enable(mut self) -> Self { + self.set_enable(true); + self + } +} + +impl UpstreamObject for SpamDnsblServer { + fn replacement_for(mut self, local: &Self) -> Option { + self.set_enable(local.enable()); + Some(self) + } + + fn without_enable(mut self) -> Self { + self.set_enable(true); + self + } +} + +impl UpstreamObject for HttpLookup { + fn replacement_for(mut self, local: &Self) -> Option { + self.enable = local.enable; + Some(self) + } + + fn without_enable(mut self) -> Self { + self.enable = true; + self + } +} + +impl UpstreamObject for SpamTag { + fn replacement_for(self, _local: &Self) -> Option { + None + } +} + +impl UpstreamObject for MemoryLookupKey {} + +impl UpstreamObject for SpamFileExtension {} + struct RuleUpdateResult { - success: usize, - already_exists: usize, + object_type: ObjectType, + added: usize, + updated: usize, + unchanged: usize, + kept: usize, failed: usize, } +impl RuleUpdateResult { + fn new(object_type: ObjectType) -> Self { + RuleUpdateResult { + object_type, + added: 0, + updated: 0, + unchanged: 0, + kept: 0, + failed: 0, + } + } + + fn has_changes(&self) -> bool { + self.added + self.updated > 0 + } +} + async fn update_spam_rules(server: &Server) -> trc::Result { let started = Instant::now(); let bundled = server.core.spam.spam_rules_url.is_none(); @@ -121,171 +198,67 @@ async fn update_spam_rules(server: &Server) -> trc::Result { } }; - let registry = server.registry(); - let mut stats: AHashMap = AHashMap::new(); + let settings = [ + apply_upstream(server, rules.rules).await?, + apply_upstream(server, rules.dnsbls).await?, + apply_upstream(server, rules.tags).await?, + apply_upstream(server, rules.file_exts).await?, + ]; + let lookups = [ + apply_upstream(server, rules.http_lookups).await?, + apply_upstream(server, rules.key_lookups).await?, + ]; - let mut reload_settings = false; - let mut reload_lookups = false; - - for rule in rules.rules { - match registry.write(RegistryWrite::insert(&rule.into())).await? { - RegistryWriteResult::Success(_) => { - stats.entry(ObjectType::SpamRule).or_default().success += 1; - reload_settings = true; - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - stats - .entry(ObjectType::SpamRule) - .or_default() - .already_exists += 1; - } - _ => { - stats.entry(ObjectType::SpamRule).or_default().failed += 1; - } + let mut reload_errors = Vec::new(); + for object in [ + settings + .iter() + .any(RuleUpdateResult::has_changes) + .then_some(ObjectType::SpamRule), + lookups + .iter() + .any(RuleUpdateResult::has_changes) + .then_some(ObjectType::MemoryLookupKey), + ] + .into_iter() + .flatten() + { + if let Err(reason) = reload_and_broadcast(server, object).await { + reload_errors.push(reason); } } - - for dnsbl in rules.dnsbls { - match registry.write(RegistryWrite::insert(&dnsbl.into())).await? { - RegistryWriteResult::Success(_) => { - stats - .entry(ObjectType::SpamDnsblServer) - .or_default() - .success += 1; - reload_settings = true; - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - stats - .entry(ObjectType::SpamDnsblServer) - .or_default() - .already_exists += 1; - } - _ => { - stats.entry(ObjectType::SpamDnsblServer).or_default().failed += 1; - } - } - } - - for tag in rules.tags { - match registry.write(RegistryWrite::insert(&tag.into())).await? { - RegistryWriteResult::Success(_) => { - stats.entry(ObjectType::SpamTag).or_default().success += 1; - reload_settings = true; - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - stats.entry(ObjectType::SpamTag).or_default().already_exists += 1; - } - _ => { - stats.entry(ObjectType::SpamTag).or_default().failed += 1; - } - } - } - - for lookup in rules.http_lookups { - match registry - .write(RegistryWrite::insert(&lookup.into())) - .await? - { - RegistryWriteResult::Success(_) => { - stats.entry(ObjectType::HttpLookup).or_default().success += 1; - reload_lookups = true; - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - stats - .entry(ObjectType::HttpLookup) - .or_default() - .already_exists += 1; - } - _ => { - stats.entry(ObjectType::HttpLookup).or_default().failed += 1; - } - } - } - - for key_lookup in rules.key_lookups { - match registry - .write(RegistryWrite::insert(&key_lookup.into())) - .await? - { - RegistryWriteResult::Success(_) => { - stats - .entry(ObjectType::MemoryLookupKey) - .or_default() - .success += 1; - reload_lookups = true; - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - stats - .entry(ObjectType::MemoryLookupKey) - .or_default() - .already_exists += 1; - } - _ => { - stats.entry(ObjectType::MemoryLookupKey).or_default().failed += 1; - } - } - } - - for ext in rules.file_exts { - match registry.write(RegistryWrite::insert(&ext.into())).await? { - RegistryWriteResult::Success(_) => { - stats - .entry(ObjectType::SpamFileExtension) - .or_default() - .success += 1; - reload_settings = true; - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - stats - .entry(ObjectType::SpamFileExtension) - .or_default() - .already_exists += 1; - } - _ => { - stats - .entry(ObjectType::SpamFileExtension) - .or_default() - .failed += 1; - } - } - } - - if reload_settings { - if let Err(err) = - Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::SpamRule))).await - { - trc::error!(err.details("Failed to reload registry after updating spam rules")); - } - server - .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload( - ObjectType::SpamRule, - ))) - .await; - } - - if reload_lookups { - if let Err(err) = - Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::MemoryLookupKey))) - .await - { - trc::error!(err.details("Failed to reload registry after updating spam rules")); - } - server - .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload( - ObjectType::MemoryLookupKey, - ))) - .await; - } - - // inbuxa: AU-1.10: what the update added, as one audit record - let added = stats + let failed: usize = settings .iter() - .filter(|(_, result)| result.success > 0) - .map(|(object_type, result)| format!("{} {}", result.success, object_type.as_str())) - .collect::>(); - if !added.is_empty() { - let mut added = added; - added.sort(); + .chain(&lookups) + .map(|result| result.failed) + .sum(); + + // inbuxa: AU-1.10: what the update changed, as one audit record + let summary = |count: fn(&RuleUpdateResult) -> usize| { + let mut parts = settings + .iter() + .chain(&lookups) + .filter(|result| count(result) > 0) + .map(|result| format!("{} {}", count(result), result.object_type.as_str())) + .collect::>(); + parts.sort(); + parts.join(", ") + }; + let details = [ + ("added", summary(|result| result.added)), + ("replaced", summary(|result| result.updated)), + ("kept as edited locally", summary(|result| result.kept)), + ("failed", summary(|result| result.failed)), + ] + .into_iter() + .filter(|(_, part)| !part.is_empty()) + .map(|(what, part)| format!("{what} {part}")) + .collect::>(); + if settings + .iter() + .chain(&lookups) + .any(RuleUpdateResult::has_changes) + { server .audit_note(inbuxa_features::audit::Record { at: std::time::SystemTime::now() @@ -301,7 +274,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { ..Default::default() }, changes: vec![], - details: Some(format!("Rules update added {}", added.join(", "))), + details: Some(format!("Rules update {}", details.join("; "))), reason: None, outcome: inbuxa_features::audit::Outcome::success(), }) @@ -310,27 +283,173 @@ async fn update_spam_rules(server: &Server) -> trc::Result { trc::event!( Spam(SpamEvent::RulesUpdated), - Details = stats + Details = settings .into_iter() - .map(|(object_type, result)| { + .chain(lookups) + .map(|result| { Value::Array(vec![ - Value::String(object_type.as_str().into()), - Value::from(result.success), - Value::from(result.already_exists), + Value::String(result.object_type.as_str().into()), + Value::from(result.added), + Value::from(result.updated), + Value::from(result.unchanged), Value::from(result.failed), + Value::from(result.kept), ]) }) .collect::>(), Elapsed = started.elapsed(), ); - // inbuxa: so the next start knows these bundled rules are in - if bundled { - spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_VERSION) - .await?; + if !reload_errors.is_empty() { + Ok(TaskResult::permanent(format!( + "Spam rules were stored but not activated ({}); fix the logged errors and run Reload settings", + reload_errors.join("; ") + ))) + } else if failed > 0 { + Ok(TaskResult::permanent(format!( + "{failed} spam filter objects failed to import or update" + ))) + } else { + // inbuxa: so the next start knows these bundled rules are in. Only + // once they all are: a failed update runs again on the next start. + if bundled { + spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED) + .await?; + } + Ok(TaskResult::Success(vec![])) + } +} + +async fn reload_and_broadcast(server: &Server, object: ObjectType) -> Result<(), String> { + match Box::pin(server.reload_registry(RegistryChange::Reload(object))).await { + Ok(result) => { + result.log(); + if result.has_errors() { + return Err(format!("{} configuration errors", result.errors.len())); + } + server + .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload( + object, + ))) + .await; + Ok(()) + } + Err(err) => { + let reason = err.to_string(); + trc::error!(err.details("Failed to reload registry after updating spam rules")); + Err(reason) + } + } +} + +async fn apply_upstream( + server: &Server, + objects: Vec, +) -> trc::Result { + let registry = server.registry(); + let mut result = RuleUpdateResult::new(T::OBJECT); + + for upstream in objects { + // inbuxa: every object the update writes is fingerprinted, and an + // existing object is replaced only while it still matches: one an + // admin edited is kept as it is. + let upstream_print = fingerprint(&upstream); + let written = Object::from(upstream.clone()); + let existing_id = match registry.write(RegistryWrite::insert(&written)).await? { + RegistryWriteResult::Success(id) => { + spam_rules::set_fingerprint(server.store(), T::OBJECT, id.id(), &upstream_print) + .await?; + result.added += 1; + continue; + } + RegistryWriteResult::PrimaryKeyConflict { existing_id, .. } + if existing_id.object() == T::OBJECT => + { + existing_id + } + RegistryWriteResult::PrimaryKeyConflict { .. } => { + result.unchanged += 1; + continue; + } + _ => { + result.failed += 1; + continue; + } + }; + + let Some(local) = registry.get(existing_id).await? else { + result.failed += 1; + continue; + }; + let revision = local.revision; + let local = T::from(local); + let local_print = fingerprint(&local); + let written_print = + spam_rules::fingerprint(server.store(), T::OBJECT, existing_id.id().id()).await?; + + if local_print == upstream_print { + // The same as upstream's. An install from before fingerprints + // gets one here, so the next release can replace it. + if written_print.as_deref() != Some(upstream_print.as_str()) { + spam_rules::set_fingerprint( + server.store(), + T::OBJECT, + existing_id.id().id(), + &upstream_print, + ) + .await?; + } + result.unchanged += 1; + continue; + } + if written_print.as_deref() != Some(local_print.as_str()) { + // Changed since the update wrote it, or never written by one. + result.kept += 1; + continue; + } + + let Some(replacement) = upstream + .replacement_for(&local) + .filter(|replacement| replacement != &local) + else { + result.unchanged += 1; + continue; + }; + + let replacement = Object::from(replacement); + let local = Object::with_revision(local.into(), revision); + match registry + .write(RegistryWrite::update( + existing_id.id(), + &replacement, + &local, + )) + .await? + { + RegistryWriteResult::Success(_) => { + spam_rules::set_fingerprint( + server.store(), + T::OBJECT, + existing_id.id().id(), + &upstream_print, + ) + .await?; + result.updated += 1 + } + _ => result.failed += 1, + } } - Ok(TaskResult::Success(vec![])) + Ok(result) +} + +/// inbuxa: a digest of an object's content, `enable` aside, in hex. +fn fingerprint(object: &T) -> String { + use sha2::Digest; + sha2::Sha256::digest(serde_json::to_vec(&object.clone().without_enable()).unwrap_or_default()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() } async fn fetch_spam_rules(server: &Server) -> Result { @@ -347,13 +466,12 @@ async fn fetch_spam_rules(server: &Server) -> Result { reason, }), }; - let rules_json: AHashMap> = - bytes.and_then(|bytes| { - serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, - reason: format!("Failed to parse spam rules JSON: {err}"), - }) - })?; + let rules_json: AHashMap> = bytes.and_then(|bytes| { + serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError { + typ: TaskFailureType::Permanent, + reason: format!("Failed to parse spam rules JSON: {err}"), + }) + })?; let mut rules = Rules::default(); for (object_type, values) in rules_json { diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index ee0ba14..f2d40ec 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -6,7 +6,7 @@ homepage = "https://inbuxa.org" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index a3ee5ab..6c30dc6 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -696,7 +696,12 @@ impl Session { .map(|a| a.as_str()) .unwrap_or_default(), ) - .with_message(parsed_message); + .with_message( + edited_message + .as_deref() + .and_then(|message| MessageParser::new().parse(message)) + .unwrap_or(parsed_message), + ); let modifications = match self.run_script(script_id, script.clone(), params).await { ScriptResult::Accept { modifications } => modifications, diff --git a/crates/smtp/src/inbound/hooks/message.rs b/crates/smtp/src/inbound/hooks/message.rs index 31bff92..eebb470 100644 --- a/crates/smtp/src/inbound/hooks/message.rs +++ b/crates/smtp/src/inbound/hooks/message.rs @@ -132,7 +132,7 @@ impl Session { name: "X-Quarantine".into(), value: "true".into(), }); - FilterResponse::accept() + continue; } }; diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 4c9c0eb..d2b6b7e 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -531,11 +531,23 @@ impl QueuedMessage { }; // Obtain remote hosts list + let mx_unvalidated = mx_config.is_some() && !tls_strategy.try_dane(); let mx_list; if let Some(mx_config) = mx_config { // Lookup MX let time = Instant::now(); - mx_list = match server.mx_lookup(domain).await { + let mx_lookup = if mx_unvalidated { + server + .core + .smtp + .resolvers + .dns + .mx_lookup(domain, Some(&server.inner.cache.dns_mx)) + .await + } else { + server.mx_lookup(domain).await + }; + mx_list = match mx_lookup { Ok(mx) => mx, Err(mail_auth::Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => { trc::event!( @@ -674,6 +686,33 @@ impl QueuedMessage { message.span_id, ); + let validated_host; + let remote_host = if mx_unvalidated && tls_strategy.try_dane() { + let time = Instant::now(); + let dnssec_status = match server.mx_lookup(domain).await { + Ok(mx) => mx.dnssec_status, + Err(mail_auth::Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => { + DnssecStatus::Indeterminate + } + Err(err) => { + trc::event!( + Delivery(DeliveryEvent::MxLookupFailed), + SpanId = message.span_id, + Domain = domain.to_string(), + CausedBy = trc::Error::from(err.clone()), + Elapsed = time.elapsed(), + ); + + last_status = Status::from_mail_auth_error(domain, err); + continue 'next_host; + } + }; + validated_host = remote_host.with_dnssec_status(dnssec_status); + &validated_host + } else { + remote_host + }; + // Obtain source and remote IPs let time = Instant::now(); let validate_addresses = server.core.smtp.resolvers.dnssec_available @@ -722,15 +761,8 @@ impl QueuedMessage { let time = Instant::now(); let strict = tls_strategy.is_dane_required(); - let (dnssec_status, dnssec_entity) = match remote_host.dnssec_status() { - DnssecStatus::Secure => match addresses_dnssec_status { - status @ (DnssecStatus::Insecure | DnssecStatus::Bogus) => { - (status, "A/AAAA") - } - _ => (DnssecStatus::Secure, "MX"), - }, - status => (status, "MX"), - }; + let (dnssec_status, dnssec_entity) = + remote_host.dane_status(addresses_dnssec_status); match dnssec_status { DnssecStatus::Secure => { diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index d5e5d95..1cff2b1 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -350,12 +350,39 @@ impl NextHop<'_> { } } - fn dnssec_status(&self) -> DnssecStatus { + pub fn dnssec_status(&self) -> DnssecStatus { match self { NextHop::MX { dnssec_status, .. } => *dnssec_status, NextHop::Relay(_) => DnssecStatus::Indeterminate, } } + + fn with_dnssec_status(&self, dnssec_status: DnssecStatus) -> Self { + match self { + NextHop::MX { + is_implicit, + host, + config, + .. + } => NextHop::MX { + is_implicit: *is_implicit, + host, + config, + dnssec_status, + }, + NextHop::Relay(relay) => NextHop::Relay(relay), + } + } + + pub fn dane_status(&self, addresses: DnssecStatus) -> (DnssecStatus, &'static str) { + match self.dnssec_status() { + DnssecStatus::Secure => match addresses { + status @ (DnssecStatus::Insecure | DnssecStatus::Bogus) => (status, "A/AAAA"), + _ => (DnssecStatus::Secure, "MX"), + }, + status => (status, "MX"), + } + } } impl DeliveryResult { diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index 18c87e0..401b0b2 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -67,6 +67,10 @@ impl SpawnQueue for mpsc::Receiver { Queue::new(core, self).start().await; }); } + + fn discard(mut self) { + tokio::spawn(async move { while self.recv().await.is_some() {} }); + } } const BACK_PRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(60); @@ -510,6 +514,7 @@ impl Recipient { pub trait SpawnQueue { fn spawn(self, core: Arc); + fn discard(self); } impl QueueStats { diff --git a/crates/spam-filter/Cargo.toml b/crates/spam-filter/Cargo.toml index 3006789..b842c2e 100644 --- a/crates/spam-filter/Cargo.toml +++ b/crates/spam-filter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spam-filter" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/spam-filter/src/analysis/url.rs b/crates/spam-filter/src/analysis/url.rs index c3b8f60..b55e7c6 100644 --- a/crates/spam-filter/src/analysis/url.rs +++ b/crates/spam-filter/src/analysis/url.rs @@ -203,7 +203,7 @@ impl SpamFilterAnalyzeUrl for Server { if !ctx.result.has_tag("URL_REDIRECTOR_NESTED") { let mut redirect_count = 1; - let mut url_redirect = Cow::Borrowed(url.element.url.as_str()); + let mut url_redirect = url.element.request_url(); while redirect_count <= 3 { match http_get_header( @@ -224,7 +224,8 @@ impl SpamFilterAnalyzeUrl for Server { ) .await { - url_redirect = Cow::Owned(location.url); + url_redirect = + Cow::Owned(location.request_url().into_owned()); redirect_count += 1; continue; } else { @@ -487,6 +488,15 @@ impl<'x> UrlParts<'x> { .is_some_and(|url| url.host.fqdn.starts_with("www.")) } + pub fn request_url(&self) -> Cow<'_, str> { + let url = self.url_original.trim(); + if self.has_scheme { + Cow::Borrowed(url) + } else { + Cow::Owned([HTTPS_SCHEME, url].concat()) + } + } + fn parse(url: &str) -> Option { url.parse::().ok().and_then(|parts| { parts @@ -505,3 +515,19 @@ impl<'x> UrlParts<'x> { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_url_keeps_case() { + let url = UrlParts::new(" https://Bit.ly/3AbCdEf "); + assert_eq!(url.url, "https://bit.ly/3abcdef"); + assert_eq!(url.request_url(), "https://Bit.ly/3AbCdEf"); + + let url = UrlParts::no_scheme("Bit.ly/3AbCdEf"); + assert_eq!(url.url, "https://bit.ly/3abcdef"); + assert_eq!(url.request_url(), "https://Bit.ly/3AbCdEf"); + } +} diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index 04b92f3..dca505a 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -259,9 +259,7 @@ impl SpamClassifier for Server { remove_entries = true; } - if trainer.last_id == 0 { - trainer.last_id = id; - } + trainer.last_id = trainer.last_id.max(id); Ok(true) }, diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 347e426..ba1c06d 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "store" -version = "0.16.23" +version = "0.16.24" edition = "2024" [dependencies] diff --git a/crates/store/src/backend/http/lookup.rs b/crates/store/src/backend/http/lookup.rs index 9d9c4ec..76282db 100644 --- a/crates/store/src/backend/http/lookup.rs +++ b/crates/store/src/backend/http/lookup.rs @@ -4,21 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{HttpStore, HttpStoreConfig}; +use crate::{Value, backend::http::HttpStoreFormat, write::now}; +use ahash::AHashMap; +use compact_str::ToCompactString; +use rand::seq::IndexedRandom; use std::{ + borrow::Cow, io::{BufRead, BufReader}, sync::{Arc, atomic::Ordering}, time::Instant, }; - -use ahash::AHashMap; -use compact_str::ToCompactString; -use rand::seq::IndexedRandom; use utils::HttpLimitResponse; -use crate::{Value, backend::http::HttpStoreFormat, write::now}; - -use super::HttpStore; - const BROWSER_USER_AGENTS: [&str; 5] = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/120.0.0.0 Safari/537.36", @@ -36,7 +34,7 @@ pub(crate) trait HttpStoreGet { impl HttpStoreGet for Arc { fn get(&self, key: &str) -> Option> { self.refresh(); - self.entries.load().get(key).cloned() + self.entries.load().get(lookup_key(key).as_ref()).cloned() } fn contains(&self, key: &str) -> bool { @@ -59,7 +57,7 @@ impl HttpStoreGet for Arc { } self.refresh(); - self.entries.load().contains_key(key) + self.entries.load().contains_key(lookup_key(key).as_ref()) } fn refresh(&self) { @@ -142,9 +140,10 @@ impl HttpStore { Box::new(&bytes[..]) }; - let mut entries = AHashMap::new(); - for (pos, line) in BufReader::new(reader).lines().enumerate() { - let line_ = line.map_err(|err| { + let entries = self + .config + .parse_entries(BufReader::new(reader)) + .map_err(|err| { trc::StoreEvent::HttpStoreError .into_err() .reason(err) @@ -153,11 +152,31 @@ impl HttpStore { .details("Failed to read line") })?; - match &self.config.format { + trc::event!( + Store(trc::StoreEvent::HttpStoreFetch), + Url = self.config.url.to_compact_string(), + Total = entries.len(), + Elapsed = time.elapsed(), + ); + + Ok(entries) + } +} + +impl HttpStoreConfig { + fn parse_entries( + &self, + reader: impl BufRead, + ) -> std::io::Result>> { + let mut entries = AHashMap::new(); + for (pos, line) in reader.lines().enumerate() { + let line_ = line?; + + match &self.format { HttpStoreFormat::List => { let line = line_.trim(); if !line.is_empty() { - entries.insert(line.to_string(), Value::Integer(1)); + entries.insert(lookup_key(line).into_owned(), Value::Integer(1)); } } HttpStoreFormat::Csv { @@ -188,12 +207,12 @@ impl HttpStore { } } else if col_num == *index_key { entry_key.push(ch); - if entry_key.len() > self.config.max_entry_size { + if entry_key.len() > self.max_entry_size { break; } } else if index_value.is_some_and(|v| col_num == v) { entry_value.push(ch); - if entry_value.len() > self.config.max_entry_size { + if entry_value.len() > self.max_entry_size { break; } } @@ -209,24 +228,122 @@ impl HttpStore { } else { Value::Integer(1) }; + let entry_key = match lookup_key(&entry_key) { + Cow::Owned(key) => key, + Cow::Borrowed(_) => entry_key, + }; entries.insert(entry_key, entry_value); } } _ => (), } - if entries.len() == self.config.max_entries { + if entries.len() == self.max_entries { break; } } - trc::event!( - Store(trc::StoreEvent::HttpStoreFetch), - Url = self.config.url.to_compact_string(), - Total = entries.len(), - Elapsed = time.elapsed(), - ); - Ok(entries) } } + +fn lookup_key(key: &str) -> Cow<'_, str> { + if key.bytes().any(|b| !b.is_ascii() || b.is_ascii_uppercase()) { + Cow::Owned(key.to_lowercase()) + } else { + Cow::Borrowed(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arc_swap::ArcSwap; + use reqwest::Client; + use std::{ + sync::atomic::{AtomicBool, AtomicU64}, + time::Duration, + }; + + fn http_store(format: HttpStoreFormat, feed: &str) -> Arc { + let config = HttpStoreConfig { + id: "test".into(), + url: "https://lists.example.org/feed".into(), + retry: 0, + refresh: 0, + timeout: Duration::from_secs(1), + gzipped: false, + max_size: 1024 * 1024, + max_entries: 100, + max_entry_size: 512, + format, + }; + let entries = config + .parse_entries(feed.as_bytes()) + .expect("feed is readable"); + + Arc::new(HttpStore { + entries: ArcSwap::from_pointee(entries), + expires: AtomicU64::new(u64::MAX), + in_flight: AtomicBool::new(false), + config, + client: Client::new(), + }) + } + + #[test] + fn list_keys_ignore_case() { + let store = http_store( + HttpStoreFormat::List, + "https://phish.example.org/Account/Verify?Token=AbC123\n\ + https://PHISH.example.net/lower\n", + ); + + assert!(store.contains("https://phish.example.org/account/verify?token=abc123")); + assert!(store.contains("https://phish.example.org/Account/Verify?Token=AbC123")); + assert!(store.contains("https://phish.example.net/lower")); + assert!(store.contains("HTTPS://PHISH.EXAMPLE.NET/LOWER")); + assert!(!store.contains("https://phish.example.org/account/verify")); + } + + #[test] + fn csv_keys_ignore_case() { + let store = http_store( + HttpStoreFormat::Csv { + index_key: 1, + index_value: None, + separator: ',', + skip_first: true, + }, + "phish_id,url,phish_detail_url\n\ + 1,\"https://phish.example.org/Login.PHP?Id=Xy\",https://phishtank.example/1\n", + ); + + assert!(store.contains("https://phish.example.org/login.php?id=xy")); + assert!(store.contains("https://phish.example.org/Login.PHP?Id=Xy")); + assert!(!store.contains("phish_id")); + assert!(!store.contains("url")); + } + + #[test] + fn csv_values_keep_case() { + let store = http_store( + HttpStoreFormat::Csv { + index_key: 0, + index_value: Some(1), + separator: ',', + skip_first: false, + }, + "Example.ORG,Some Value\n", + ); + + assert_eq!( + store.get("example.org"), + Some(Value::Text("Some Value".into())) + ); + assert_eq!( + store.get("EXAMPLE.org"), + Some(Value::Text("Some Value".into())) + ); + } +} diff --git a/crates/store/src/backend/mysql/mod.rs b/crates/store/src/backend/mysql/mod.rs index d1c0b8c..8fe4365 100644 --- a/crates/store/src/backend/mysql/mod.rs +++ b/crates/store/src/backend/mysql/mod.rs @@ -99,7 +99,10 @@ pub(crate) fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::MysqlError.reason(err) } +const ER_UNKNOWN_ERROR: u16 = 1105; +const ER_TRANS_CACHE_FULL: u16 = 1197; const ER_LOCK_WAIT_TIMEOUT: u16 = 1205; +const ER_LOCK_TABLE_FULL: u16 = 1206; const ER_STATEMENT_TIMEOUT: u16 = 1969; const ER_QUERY_TIMEOUT: u16 = 3024; @@ -116,6 +119,17 @@ pub(crate) fn is_timeout_error(err: &mysql_async::Error) -> bool { ) } +#[inline(always)] +pub(crate) fn is_chunk_too_large_error(err: &mysql_async::Error) -> bool { + is_timeout_error(err) + || matches!(err, mysql_async::Error::Server(err) + if matches!( + err.code, + ER_UNKNOWN_ERROR | ER_TRANS_CACHE_FULL | ER_LOCK_TABLE_FULL + ) + ) +} + impl SearchIndex { pub fn mysql_table(&self) -> &'static str { match self { diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index 906ce66..7b596ba 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -11,7 +11,7 @@ use crate::{ MAX_TOKEN_LENGTH, mysql::{ DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, bounded, - into_error, is_timeout_error, + into_error, is_chunk_too_large_error, }, }, search::{ @@ -123,14 +123,6 @@ impl MysqlStore { let mut conn = self.conn().await?; let limit = self.timeouts.maintenance; let result = tokio::time::timeout(limit, async { - let s = conn.prep(&query).await.map_err(into_error)?; - - match conn.exec_drop(s, params.clone()).await { - Ok(_) => return Ok(conn.affected_rows()), - Err(err) if is_timeout_error(&err) => (), - Err(err) => return Err(into_error(err)), - } - let mut chunk_size = DELETE_CHUNK_SIZE; let mut deleted = 0; @@ -144,13 +136,14 @@ impl MysqlStore { match conn.exec_drop(&s, params.clone()).await { Ok(_) => { let affected = conn.affected_rows(); - if affected == 0 { + deleted += affected; + if affected < chunk_size as u64 { return Ok(deleted); } - deleted += affected; } Err(err) - if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => + if is_chunk_too_large_error(&err) + && chunk_size > MIN_DELETE_CHUNK_SIZE => { chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); break; diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index d869bc0..ea6ae55 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -7,7 +7,8 @@ */ use super::{ - DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, bounded, into_error, is_timeout_error, + DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, bounded, into_error, + is_chunk_too_large_error, }; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, @@ -416,12 +417,6 @@ impl MysqlStore { .await .map_err(into_error)?; - match conn.exec_drop(&delete, (&from, &to)).await { - Ok(_) => return Ok(()), - Err(err) if is_timeout_error(&err) => (), - Err(err) => return Err(into_error(err)), - } - let mut chunk_size = DELETE_CHUNK_SIZE; loop { @@ -438,7 +433,10 @@ impl MysqlStore { .await { Ok(next) => next, - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { + Err(err) + if is_chunk_too_large_error(&err) + && chunk_size > MIN_DELETE_CHUNK_SIZE => + { chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); break; } @@ -450,7 +448,10 @@ impl MysqlStore { .await { Ok(_) => (), - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { + Err(err) + if is_chunk_too_large_error(&err) + && chunk_size > MIN_DELETE_CHUNK_SIZE => + { chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); break; } @@ -477,7 +478,7 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { match conn.exec_drop(&s, ()).await { Ok(_) => return Ok(()), - Err(err) if is_timeout_error(&err) => (), + Err(err) if is_chunk_too_large_error(&err) => (), Err(err) => return Err(into_error(err)), } @@ -505,7 +506,9 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { loop { let next = match conn.exec_first::, _, _>(&boundary, (&from,)).await { Ok(next) => next, - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { + Err(err) + if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => + { chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); break; } @@ -519,7 +522,9 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { match result { Ok(_) => (), - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { + Err(err) + if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => + { chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); break; } diff --git a/crates/store/src/backend/redis/lookup.rs b/crates/store/src/backend/redis/lookup.rs index f1c9cc4..5a31be9 100644 --- a/crates/store/src/backend/redis/lookup.rs +++ b/crates/store/src/backend/redis/lookup.rs @@ -9,16 +9,7 @@ use super::{RedisPool, RedisStore, into_error}; use crate::{Deserialize, write::now}; use deadpool::managed::{Manager, Object, Pool}; -use redis::{AsyncCommands, RedisError, RedisResult, RetryMethod, Script}; -use std::sync::LazyLock; - -static INCR_EXPIRE: LazyLock